2024-04-05T02:37:50.194 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-05T02:37:50.194 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --owner scheduled_teuthology@teuthology --name teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239 --verbose -- /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239/orig.config.yaml 2024-04-05T02:37:50.256 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-05T02:37:50.364 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7641239' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: 9f192350ba5fe6d0a365b251a8c2af61f56a5fa6 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 8639 sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 27/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 9f192350ba5fe6d0a365b251a8c2af61f56a5fa6 targets: smithi033.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGb6Z0f1svxcqtuGH/EwOxinIHa3wn5YuwPddKd3VEnYkXq3N6mhfHs3NKcYjqW65FVgdghso4GaRJxivQAwOJc= smithi151.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCFjpiIx5qJZ+E0EHmLE7Ann6VQXu5/03zhZiPK+z0IpNCeJN9EIUQ8EFqAWSt1Dou+1kGAzgxWvEmPt4nFxbCU= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-05_02:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-05T02:37:50.364 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa; will attempt to use it 2024-04-05T02:37:50.365 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks 2024-04-05T02:37:50.365 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-05T02:37:50.366 INFO:teuthology.task.internal:Checking packages... 2024-04-05T02:37:50.391 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '0028525c15e19309c20289c609e4fb9d0b12dd06' 2024-04-05T02:37:50.391 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:37:50.391 INFO:teuthology.packaging:ref: None 2024-04-05T02:37:50.391 INFO:teuthology.packaging:tag: None 2024-04-05T02:37:50.391 INFO:teuthology.packaging:branch: squid 2024-04-05T02:37:50.391 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:37:50.391 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-05T02:37:50.591 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2013.g0028525c 2024-04-05T02:37:50.592 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-05T02:37:50.599 INFO:teuthology.task.internal:no buildpackages task found 2024-04-05T02:37:50.599 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-05T02:37:50.605 INFO:teuthology.task.internal:Saving configuration 2024-04-05T02:37:50.630 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-05T02:37:50.636 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-05T02:37:50.660 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi033.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:32:33.016485', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGb6Z0f1svxcqtuGH/EwOxinIHa3wn5YuwPddKd3VEnYkXq3N6mhfHs3NKcYjqW65FVgdghso4GaRJxivQAwOJc='} 2024-04-05T02:37:50.680 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:32:33.017904', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCFjpiIx5qJZ+E0EHmLE7Ann6VQXu5/03zhZiPK+z0IpNCeJN9EIUQ8EFqAWSt1Dou+1kGAzgxWvEmPt4nFxbCU='} 2024-04-05T02:37:50.680 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-05T02:37:50.686 INFO:teuthology.task.internal:roles: ubuntu@smithi033.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-05T02:37:50.686 INFO:teuthology.task.internal:roles: ubuntu@smithi151.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-05T02:37:50.686 INFO:teuthology.run_tasks:Running task console_log... 2024-04-05T02:37:50.762 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f1d3370f0d0>, signals=[15]) 2024-04-05T02:37:50.763 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-05T02:37:50.769 INFO:teuthology.task.internal:Opening connections... 2024-04-05T02:37:50.769 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi033.front.sepia.ceph.com 2024-04-05T02:37:50.771 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:37:50.884 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi151.front.sepia.ceph.com 2024-04-05T02:37:50.885 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:37:50.974 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-05T02:37:50.980 DEBUG:teuthology.orchestra.run.smithi033:> uname -m 2024-04-05T02:37:50.997 INFO:teuthology.orchestra.run.smithi033.stdout:x86_64 2024-04-05T02:37:50.998 DEBUG:teuthology.orchestra.run.smithi033:> cat /etc/os-release 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:NAME="CentOS Stream" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION="9" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:ID="centos" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:ID_LIKE="rhel fedora" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION_ID="9" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:PLATFORM_ID="platform:el9" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:ANSI_COLOR="0;31" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:LOGO="fedora-logo-icon" 2024-04-05T02:37:51.056 INFO:teuthology.orchestra.run.smithi033.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-05T02:37:51.057 INFO:teuthology.orchestra.run.smithi033.stdout:HOME_URL="https://centos.org/" 2024-04-05T02:37:51.057 INFO:teuthology.orchestra.run.smithi033.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-05T02:37:51.057 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-05T02:37:51.057 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-05T02:37:51.057 INFO:teuthology.lock.ops:Updating smithi033.front.sepia.ceph.com on lock server 2024-04-05T02:37:51.079 DEBUG:teuthology.orchestra.run.smithi151:> uname -m 2024-04-05T02:37:51.097 INFO:teuthology.orchestra.run.smithi151.stdout:x86_64 2024-04-05T02:37:51.097 DEBUG:teuthology.orchestra.run.smithi151:> cat /etc/os-release 2024-04-05T02:37:51.154 INFO:teuthology.orchestra.run.smithi151.stdout:NAME="CentOS Stream" 2024-04-05T02:37:51.154 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION="9" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:ID="centos" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:ID_LIKE="rhel fedora" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION_ID="9" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:PLATFORM_ID="platform:el9" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:ANSI_COLOR="0;31" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:LOGO="fedora-logo-icon" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:HOME_URL="https://centos.org/" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-05T02:37:51.155 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-05T02:37:51.156 INFO:teuthology.lock.ops:Updating smithi151.front.sepia.ceph.com on lock server 2024-04-05T02:37:51.179 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-05T02:37:51.188 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-05T02:37:51.193 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-05T02:37:51.193 DEBUG:teuthology.orchestra.run.smithi033:> test '!' -e /home/ubuntu/cephtest 2024-04-05T02:37:51.196 DEBUG:teuthology.orchestra.run.smithi151:> test '!' -e /home/ubuntu/cephtest 2024-04-05T02:37:51.212 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-05T02:37:51.217 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-05T02:37:51.217 DEBUG:teuthology.orchestra.run.smithi033:> test -z $(ls -A /var/lib/ceph) 2024-04-05T02:37:51.253 DEBUG:teuthology.orchestra.run.smithi151:> test -z $(ls -A /var/lib/ceph) 2024-04-05T02:37:51.275 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-05T02:37:51.331 INFO:teuthology.run_tasks:Running task kernel... 2024-04-05T02:37:51.345 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-05T02:37:51.345 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-05T02:37:51.345 DEBUG:teuthology.orchestra.run.smithi033:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-05T02:37:51.346 DEBUG:teuthology.orchestra.run.smithi151:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-05T02:37:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:37:51.362 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2024-04-05T02:37:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:37:51.363 DEBUG:teuthology.orchestra.run.smithi033:> uname -r 2024-04-05T02:37:51.419 INFO:teuthology.orchestra.run.smithi151.stdout:5.14.0-361.el9.x86_64 2024-04-05T02:37:51.419 INFO:teuthology.task.kernel:Running kernel on smithi151: 5.14.0-361.el9.x86_64 2024-04-05T02:37:51.419 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum install -y kernel 2024-04-05T02:37:51.421 INFO:teuthology.orchestra.run.smithi033.stdout:5.14.0-361.el9.x86_64 2024-04-05T02:37:51.421 INFO:teuthology.task.kernel:Running kernel on smithi033: 5.14.0-361.el9.x86_64 2024-04-05T02:37:51.421 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum install -y kernel 2024-04-05T02:37:53.627 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS-9 - AppStream 14 MB/s | 6.7 MB 00:00 2024-04-05T02:37:53.695 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS-9 - AppStream 13 MB/s | 6.7 MB 00:00 2024-04-05T02:37:55.077 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS-9 - BaseOS 6.8 MB/s | 2.0 MB 00:00 2024-04-05T02:37:55.354 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS-9 - BaseOS 3.7 MB/s | 2.0 MB 00:00 2024-04-05T02:37:56.378 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-05T02:37:58.304 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - AppStream 22 MB/s | 19 MB 00:00 2024-04-05T02:38:02.516 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - CRB 9.1 MB/s | 6.1 MB 00:00 2024-04-05T02:38:04.424 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - BaseOS 913 kB/s | 8.0 MB 00:09 2024-04-05T02:38:04.980 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - Extras packages 12 kB/s | 16 kB 00:01 2024-04-05T02:38:05.841 INFO:teuthology.orchestra.run.smithi151.stdout:Extra Packages for Enterprise Linux 28 MB/s | 21 MB 00:00 2024-04-05T02:38:10.940 INFO:teuthology.orchestra.run.smithi151.stdout:lab-extras 8.2 kB/s | 1.7 kB 00:00 2024-04-05T02:38:13.197 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-05T02:38:13.197 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-05T02:38:13.303 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:38:13.318 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:38:13.318 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:38:13.318 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:38:13.318 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:38:13.318 INFO:teuthology.orchestra.run.smithi151.stdout: kernel x86_64 5.14.0-435.el9 CentOS-BaseOS 212 k 2024-04-05T02:38:13.318 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-core x86_64 5.14.0-435.el9 CentOS-BaseOS 15 M 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules x86_64 5.14.0-435.el9 CentOS-BaseOS 34 M 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules-core x86_64 5.14.0-435.el9 CentOS-BaseOS 28 M 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout:Install 4 Packages 2024-04-05T02:38:13.319 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:38:13.320 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 77 M 2024-04-05T02:38:13.320 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 123 M 2024-04-05T02:38:13.320 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:38:13.503 INFO:teuthology.orchestra.run.smithi151.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.1 MB/s | 212 kB 00:00 2024-04-05T02:38:13.925 INFO:teuthology.orchestra.run.smithi151.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 25 MB/s | 15 MB 00:00 2024-04-05T02:38:14.542 INFO:teuthology.orchestra.run.smithi151.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 28 MB/s | 34 MB 00:01 2024-04-05T02:38:14.834 INFO:teuthology.orchestra.run.smithi151.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 21 MB/s | 28 MB 00:01 2024-04-05T02:38:14.837 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:38:14.837 INFO:teuthology.orchestra.run.smithi151.stdout:Total 50 MB/s | 77 MB 00:01 2024-04-05T02:38:14.837 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:38:15.187 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:38:15.188 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:38:16.034 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:38:16.035 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:38:18.354 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:38:18.950 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:38:19.043 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:38:20.565 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - AppStream 1.3 MB/s | 19 MB 00:15 2024-04-05T02:38:21.811 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:38:21.953 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:38:25.551 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - CRB 4.7 MB/s | 6.1 MB 00:01 2024-04-05T02:38:26.701 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:38:26.802 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:38:27.216 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 16 kB 00:00 2024-04-05T02:38:29.395 INFO:teuthology.orchestra.run.smithi033.stdout:Extra Packages for Enterprise Linux 10 MB/s | 21 MB 00:02 2024-04-05T02:38:31.711 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:38:34.625 INFO:teuthology.orchestra.run.smithi033.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-04-05T02:38:36.983 INFO:teuthology.orchestra.run.smithi033.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-05T02:38:36.983 INFO:teuthology.orchestra.run.smithi033.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-05T02:38:37.115 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:38:37.121 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:38:37.121 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:38:37.121 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:38:37.121 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:38:37.121 INFO:teuthology.orchestra.run.smithi033.stdout: kernel x86_64 5.14.0-435.el9 CentOS-BaseOS 212 k 2024-04-05T02:38:37.121 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:38:37.121 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-core x86_64 5.14.0-435.el9 CentOS-BaseOS 15 M 2024-04-05T02:38:37.122 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules x86_64 5.14.0-435.el9 CentOS-BaseOS 34 M 2024-04-05T02:38:37.122 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-core x86_64 5.14.0-435.el9 CentOS-BaseOS 28 M 2024-04-05T02:38:37.122 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:38:37.122 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:38:37.122 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:38:37.122 INFO:teuthology.orchestra.run.smithi033.stdout:Install 4 Packages 2024-04-05T02:38:37.122 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:38:37.123 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 77 M 2024-04-05T02:38:37.123 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 123 M 2024-04-05T02:38:37.123 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:38:37.277 INFO:teuthology.orchestra.run.smithi033.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.3 MB/s | 212 kB 00:00 2024-04-05T02:38:37.769 INFO:teuthology.orchestra.run.smithi033.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 23 MB/s | 15 MB 00:00 2024-04-05T02:38:38.386 INFO:teuthology.orchestra.run.smithi033.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 27 MB/s | 34 MB 00:01 2024-04-05T02:38:38.644 INFO:teuthology.orchestra.run.smithi033.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 20 MB/s | 28 MB 00:01 2024-04-05T02:38:38.648 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:38:38.648 INFO:teuthology.orchestra.run.smithi033.stdout:Total 50 MB/s | 77 MB 00:01 2024-04-05T02:38:38.648 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:38:39.085 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:38:39.085 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:38:39.950 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:38:39.951 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:38:42.386 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:38:42.986 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:38:43.078 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:38:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:38:46.105 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:38:51.184 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:38:51.222 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:38:56.316 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:01.200 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:01.200 INFO:teuthology.orchestra.run.smithi151.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-05T02:39:01.200 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:39:01.244 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:01.611 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:01.611 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:39:01.611 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:39:01.611 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:39:02.037 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:39:02.384 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:39:02.489 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:39:02.490 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:39:02.490 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-05T02:39:02.490 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-05T02:39:02.490 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-05T02:39:02.490 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-05T02:39:02.490 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi151.front.sepia.ceph.com, path=None, version=distro) 2024-04-05T02:39:02.491 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:39:02.571 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:39:02.571 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:39:02.571 DEBUG:teuthology.orchestra.run.smithi151:> sudo rpm -qi grub2-tools 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Name : grub2-tools 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Epoch : 1 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Version : 2.06 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Release : 68.el9 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Architecture: x86_64 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Group : Unspecified 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Size : 8264254 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:License : GPLv3+ 2024-04-05T02:39:02.612 INFO:teuthology.orchestra.run.smithi151.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:Packager : builder@centos.org 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:Vendor : CentOS 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:Summary : Support tools for GRUB. 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:Description : 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout:hardware devices. 2024-04-05T02:39:02.613 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:39:02.614 INFO:teuthology.orchestra.run.smithi151.stdout:This subpackage provides tools for support of all platforms. 2024-04-05T02:39:02.614 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-05T02:39:02.615 INFO:teuthology.task.kernel:Updating grub on smithi151 to boot 5.14.0-435.el9.x86_64 2024-04-05T02:39:02.615 DEBUG:teuthology.orchestra.run.smithi151:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-05T02:39:03.180 INFO:teuthology.orchestra.run.smithi151.stderr:Generating grub configuration file ... 2024-04-05T02:39:04.632 INFO:teuthology.orchestra.run.smithi151.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-05T02:39:04.662 INFO:teuthology.orchestra.run.smithi151.stderr:done 2024-04-05T02:39:04.671 DEBUG:teuthology.orchestra.run.smithi151:> mktemp 2024-04-05T02:39:04.686 INFO:teuthology.orchestra.run.smithi151.stdout:/tmp/tmp.2kA2E9Zyi1 2024-04-05T02:39:04.686 DEBUG:teuthology.orchestra.run.smithi151:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.2kA2E9Zyi1 2024-04-05T02:39:04.757 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 0666 /tmp/tmp.2kA2E9Zyi1 2024-04-05T02:39:04.897 DEBUG:teuthology.orchestra.remote:smithi151:/tmp/tmp.2kA2E9Zyi1 is 6KB 2024-04-05T02:39:04.945 DEBUG:teuthology.orchestra.run.smithi151:> rm -fr /tmp/tmp.2kA2E9Zyi1 2024-04-05T02:39:04.959 DEBUG:teuthology.orchestra.run.smithi151:> sudo /bin/ls /boot/loader/entries || true 2024-04-05T02:39:05.024 INFO:teuthology.orchestra.run.smithi151.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-05T02:39:05.024 INFO:teuthology.orchestra.run.smithi151.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-05T02:39:05.024 INFO:teuthology.orchestra.run.smithi151.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-05T02:39:05.024 INFO:teuthology.orchestra.run.smithi151.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-05T02:39:05.025 DEBUG:teuthology.orchestra.run.smithi151:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-05T02:39:05.169 DEBUG:teuthology.orchestra.run.smithi151:> sudo shutdown -r now 2024-04-05T02:39:27.029 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:27.029 INFO:teuthology.orchestra.run.smithi033.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-05T02:39:27.029 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:39:27.075 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:27.485 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:27.486 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:39:27.486 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:39:27.486 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-05T02:39:27.909 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:39:27.910 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:39:28.235 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:39:28.331 INFO:teuthology.orchestra.run.smithi033.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:39:28.331 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:39:28.331 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-05T02:39:28.331 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-05T02:39:28.331 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-05T02:39:28.332 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-05T02:39:28.332 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi033.front.sepia.ceph.com, path=None, version=distro) 2024-04-05T02:39:28.332 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:39:28.414 INFO:teuthology.orchestra.run.smithi033.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:39:28.415 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:39:28.415 DEBUG:teuthology.orchestra.run.smithi033:> sudo rpm -qi grub2-tools 2024-04-05T02:39:28.452 INFO:teuthology.orchestra.run.smithi033.stdout:Name : grub2-tools 2024-04-05T02:39:28.452 INFO:teuthology.orchestra.run.smithi033.stdout:Epoch : 1 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Version : 2.06 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Release : 68.el9 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Architecture: x86_64 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Group : Unspecified 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Size : 8264254 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:License : GPLv3+ 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-05T02:39:28.453 INFO:teuthology.orchestra.run.smithi033.stdout:Packager : builder@centos.org 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:Vendor : CentOS 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:Summary : Support tools for GRUB. 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:Description : 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:hardware devices. 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:39:28.454 INFO:teuthology.orchestra.run.smithi033.stdout:This subpackage provides tools for support of all platforms. 2024-04-05T02:39:28.455 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-05T02:39:28.455 INFO:teuthology.task.kernel:Updating grub on smithi033 to boot 5.14.0-435.el9.x86_64 2024-04-05T02:39:28.455 DEBUG:teuthology.orchestra.run.smithi033:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-05T02:39:29.010 INFO:teuthology.orchestra.run.smithi033.stderr:Generating grub configuration file ... 2024-04-05T02:39:30.491 INFO:teuthology.orchestra.run.smithi033.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-05T02:39:30.531 INFO:teuthology.orchestra.run.smithi033.stderr:done 2024-04-05T02:39:30.531 DEBUG:teuthology.orchestra.run.smithi033:> mktemp 2024-04-05T02:39:30.546 INFO:teuthology.orchestra.run.smithi033.stdout:/tmp/tmp.3OaOIsyuxn 2024-04-05T02:39:30.547 DEBUG:teuthology.orchestra.run.smithi033:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.3OaOIsyuxn 2024-04-05T02:39:30.619 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 0666 /tmp/tmp.3OaOIsyuxn 2024-04-05T02:39:30.762 DEBUG:teuthology.orchestra.remote:smithi033:/tmp/tmp.3OaOIsyuxn is 6KB 2024-04-05T02:39:30.811 DEBUG:teuthology.orchestra.run.smithi033:> rm -fr /tmp/tmp.3OaOIsyuxn 2024-04-05T02:39:30.825 DEBUG:teuthology.orchestra.run.smithi033:> sudo /bin/ls /boot/loader/entries || true 2024-04-05T02:39:30.894 INFO:teuthology.orchestra.run.smithi033.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-05T02:39:30.894 INFO:teuthology.orchestra.run.smithi033.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-05T02:39:30.895 INFO:teuthology.orchestra.run.smithi033.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-05T02:39:30.895 INFO:teuthology.orchestra.run.smithi033.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-05T02:39:30.896 DEBUG:teuthology.orchestra.run.smithi033:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-05T02:39:31.043 DEBUG:teuthology.orchestra.run.smithi033:> sudo shutdown -r now 2024-04-05T02:39:35.174 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-05T02:39:35.174 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-05T02:39:35.175 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:40:01.073 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-05T02:40:01.074 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-04-05T02:40:01.074 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:40:09.758 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.151 2024-04-05T02:40:18.768 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-05T02:40:18.768 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:40:37.151 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.151 2024-04-05T02:40:49.162 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-05T02:40:49.163 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:40:52.694 DEBUG:teuthology.orchestra.run.smithi151:> true 2024-04-05T02:40:53.094 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-05T02:40:53.095 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-435.el9.x86_64"... 2024-04-05T02:40:53.095 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2024-04-05T02:40:53.109 INFO:teuthology.orchestra.run.smithi151.stdout:5.14.0-435.el9.x86_64 2024-04-05T02:40:53.109 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-05T02:40:53.109 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-05T02:40:53.109 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-05T02:40:54.113 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-05T02:40:54.113 DEBUG:teuthology.orchestra.run.smithi151:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-05T02:40:54.201 INFO:teuthology.orchestra.run.smithi151.stdout:ttyS1 2024-04-05T02:40:54.233 DEBUG:teuthology.parallel:result is None 2024-04-05T02:41:01.076 DEBUG:teuthology.orchestra.remote:timed out 2024-04-05T02:41:10.078 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-04-05T02:41:10.079 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:41:10.426 DEBUG:teuthology.orchestra.run.smithi033:> true 2024-04-05T02:41:10.798 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-04-05T02:41:10.798 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-435.el9.x86_64"... 2024-04-05T02:41:10.798 DEBUG:teuthology.orchestra.run.smithi033:> uname -r 2024-04-05T02:41:10.814 INFO:teuthology.orchestra.run.smithi033.stdout:5.14.0-435.el9.x86_64 2024-04-05T02:41:10.814 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-05T02:41:10.814 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-05T02:41:10.814 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-05T02:41:11.815 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-05T02:41:11.815 DEBUG:teuthology.orchestra.run.smithi033:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-05T02:41:11.889 INFO:teuthology.orchestra.run.smithi033.stdout:ttyS1 2024-04-05T02:41:11.944 DEBUG:teuthology.parallel:result is None 2024-04-05T02:41:11.945 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-05T02:41:11.952 INFO:teuthology.task.internal:Creating test directory... 2024-04-05T02:41:11.952 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-05T02:41:11.954 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-05T02:41:11.973 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-05T02:41:12.065 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-05T02:41:12.073 INFO:teuthology.task.internal:Creating archive directory... 2024-04-05T02:41:12.073 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-05T02:41:12.076 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-05T02:41:12.117 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-05T02:41:12.124 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-05T02:41:12.124 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-05T02:41:12.139 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-05T02:41:12.177 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:41:12.184 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:41:12.192 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:41:12.198 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:41:12.200 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-05T02:41:12.207 INFO:teuthology.task.internal:Configuring sudo... 2024-04-05T02:41:12.207 DEBUG:teuthology.orchestra.run.smithi033:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-05T02:41:12.236 DEBUG:teuthology.orchestra.run.smithi151:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-05T02:41:12.271 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-05T02:41:12.280 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-05T02:41:12.280 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-05T02:41:12.307 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-05T02:41:12.326 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:41:12.409 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:41:12.461 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:41:12.548 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:41:12.591 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:41:12.591 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-05T02:41:12.663 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:41:12.708 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:41:12.771 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:41:12.816 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:41:12.859 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:41:12.860 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-05T02:41:12.928 DEBUG:teuthology.orchestra.run.smithi033:> sudo service rsyslog restart 2024-04-05T02:41:12.930 DEBUG:teuthology.orchestra.run.smithi151:> sudo service rsyslog restart 2024-04-05T02:41:12.977 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T02:41:13.015 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T02:41:13.275 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-05T02:41:13.283 INFO:teuthology.task.internal:Starting timer... 2024-04-05T02:41:13.284 INFO:teuthology.run_tasks:Running task pcp... 2024-04-05T02:41:13.294 INFO:teuthology.run_tasks:Running task selinux... 2024-04-05T02:41:13.303 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-05T02:41:13.303 DEBUG:teuthology.orchestra.run.smithi033:> sudo service auditd rotate 2024-04-05T02:41:13.369 INFO:teuthology.orchestra.run.smithi033.stdout:Rotating logs: 2024-04-05T02:41:13.371 DEBUG:teuthology.orchestra.run.smithi151:> sudo service auditd rotate 2024-04-05T02:41:13.428 INFO:teuthology.orchestra.run.smithi151.stdout:Rotating logs: 2024-04-05T02:41:13.430 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-05T02:41:13.430 DEBUG:teuthology.orchestra.run.smithi033:> /usr/sbin/getenforce 2024-04-05T02:41:13.455 INFO:teuthology.orchestra.run.smithi033.stdout:Permissive 2024-04-05T02:41:13.455 DEBUG:teuthology.orchestra.run.smithi151:> /usr/sbin/getenforce 2024-04-05T02:41:13.482 INFO:teuthology.orchestra.run.smithi151.stdout:Permissive 2024-04-05T02:41:13.482 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi033.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi151.front.sepia.ceph.com': 'permissive'} 2024-04-05T02:41:13.483 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T02:41:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:41:13.525 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T02:41:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:41:13.554 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-05T02:41:13.554 DEBUG:teuthology.orchestra.run.smithi033:> sudo /usr/sbin/setenforce permissive 2024-04-05T02:41:13.599 DEBUG:teuthology.orchestra.run.smithi151:> sudo /usr/sbin/setenforce permissive 2024-04-05T02:41:13.632 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-05T02:41:13.641 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-05T02:41:13.659 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-05T02:41:13.923 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-05T02:41:13.945 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-05T02:41:13.947 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi033.front.sepia.ceph.com,smithi151.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-05T02:45:57.395 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi033.front.sepia.ceph.com'), Remote(name='ubuntu@smithi151.front.sepia.ceph.com')] 2024-04-05T02:45:57.396 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-04-05T02:45:57.397 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:45:57.490 DEBUG:teuthology.orchestra.run.smithi033:> true 2024-04-05T02:45:57.565 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-04-05T02:45:57.565 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-05T02:45:57.566 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:45:57.656 DEBUG:teuthology.orchestra.run.smithi151:> true 2024-04-05T02:45:57.693 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-05T02:45:57.694 INFO:teuthology.run_tasks:Running task clock... 2024-04-05T02:45:57.707 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-05T02:45:57.707 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-05T02:45:57.708 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T02:45:57.710 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-05T02:45:57.710 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T02:45:57.738 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-05T02:45:57.755 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-05T02:45:57.764 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-05T02:45:57.781 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-05T02:45:57.785 INFO:teuthology.orchestra.run.smithi033.stderr:sudo: ntpd: command not found 2024-04-05T02:45:57.798 INFO:teuthology.orchestra.run.smithi033.stdout:506 Cannot talk to daemon 2024-04-05T02:45:57.810 INFO:teuthology.orchestra.run.smithi151.stderr:sudo: ntpd: command not found 2024-04-05T02:45:57.814 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-05T02:45:57.824 INFO:teuthology.orchestra.run.smithi151.stdout:506 Cannot talk to daemon 2024-04-05T02:45:57.830 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-05T02:45:57.840 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-05T02:45:57.857 INFO:teuthology.orchestra.run.smithi151.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-05T02:45:57.886 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-04-05T02:45:57.889 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T02:45:57.889 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-04-05T02:45:57.890 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.890 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.890 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.890 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.908 INFO:teuthology.orchestra.run.smithi151.stderr:bash: line 1: ntpq: command not found 2024-04-05T02:45:57.911 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T02:45:57.911 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2024-04-05T02:45:57.911 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.911 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.912 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.912 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:45:57.913 INFO:teuthology.run_tasks:Running task install... 2024-04-05T02:45:57.921 DEBUG:teuthology.task.install:project ceph 2024-04-05T02:45:57.921 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06'}} 2024-04-05T02:45:57.921 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06'} 2024-04-05T02:45:57.921 INFO:teuthology.task.install:Using flavor: default 2024-04-05T02:45:57.926 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-05T02:45:57.926 INFO:teuthology.task.install:extra packages: [] 2024-04-05T02:45:57.927 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06', 'tag': None, 'wait_for_package': False} 2024-04-05T02:45:57.927 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:45:57.927 INFO:teuthology.packaging:ref: None 2024-04-05T02:45:57.927 INFO:teuthology.packaging:tag: None 2024-04-05T02:45:57.927 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:45:57.927 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:45:57.928 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:45:57.930 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06', 'tag': None, 'wait_for_package': False} 2024-04-05T02:45:57.930 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:45:57.930 INFO:teuthology.packaging:ref: None 2024-04-05T02:45:57.930 INFO:teuthology.packaging:tag: None 2024-04-05T02:45:57.930 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:45:57.930 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:45:57.930 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:45:58.259 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/ 2024-04-05T02:45:58.259 INFO:teuthology.task.install.rpm:Package version is 17.2.7-815.g93c790e1 2024-04-05T02:45:58.265 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/ 2024-04-05T02:45:58.266 INFO:teuthology.task.install.rpm:Package version is 17.2.7-815.g93c790e1 2024-04-05T02:45:58.371 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-05T02:45:58.371 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:45:58.371 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-05T02:45:58.400 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-05T02:45:58.400 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:45:58.400 INFO:teuthology.packaging:ref: None 2024-04-05T02:45:58.400 INFO:teuthology.packaging:tag: None 2024-04-05T02:45:58.400 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:45:58.400 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:45:58.400 DEBUG:teuthology.orchestra.run.smithi033:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-05T02:45:58.468 DEBUG:teuthology.orchestra.run.smithi033:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-05T02:45:58.483 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-05T02:45:58.483 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:45:58.483 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-05T02:45:58.512 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-05T02:45:58.512 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:45:58.512 INFO:teuthology.packaging:ref: None 2024-04-05T02:45:58.512 INFO:teuthology.packaging:tag: None 2024-04-05T02:45:58.512 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:45:58.512 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:45:58.512 DEBUG:teuthology.orchestra.run.smithi151:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-05T02:45:58.549 DEBUG:teuthology.orchestra.run.smithi033:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-05T02:45:58.580 DEBUG:teuthology.orchestra.run.smithi151:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-05T02:45:58.617 INFO:teuthology.orchestra.run.smithi033.stdout:check_obsoletes = 1 2024-04-05T02:45:58.618 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum clean all 2024-04-05T02:45:58.660 DEBUG:teuthology.orchestra.run.smithi151:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-05T02:45:58.727 INFO:teuthology.orchestra.run.smithi151.stdout:check_obsoletes = 1 2024-04-05T02:45:58.729 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum clean all 2024-04-05T02:45:58.869 INFO:teuthology.orchestra.run.smithi033.stdout:74 files removed 2024-04-05T02:45:58.898 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-radosgw 2024-04-05T02:45:58.970 INFO:teuthology.orchestra.run.smithi151.stdout:74 files removed 2024-04-05T02:45:58.998 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-radosgw 2024-04-05T02:45:59.520 INFO:teuthology.orchestra.run.smithi033.stdout:ceph packages for x86_64 244 kB/s | 77 kB 00:00 2024-04-05T02:45:59.607 INFO:teuthology.orchestra.run.smithi151.stdout:ceph packages for x86_64 250 kB/s | 77 kB 00:00 2024-04-05T02:45:59.904 INFO:teuthology.orchestra.run.smithi033.stdout:ceph noarch packages 28 kB/s | 9.2 kB 00:00 2024-04-05T02:45:59.974 INFO:teuthology.orchestra.run.smithi151.stdout:ceph noarch packages 29 kB/s | 9.2 kB 00:00 2024-04-05T02:46:00.279 INFO:teuthology.orchestra.run.smithi033.stdout:ceph source packages 5.3 kB/s | 1.8 kB 00:00 2024-04-05T02:46:00.391 INFO:teuthology.orchestra.run.smithi151.stdout:ceph source packages 4.7 kB/s | 1.8 kB 00:00 2024-04-05T02:46:00.849 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS-9 - AppStream 13 MB/s | 6.7 MB 00:00 2024-04-05T02:46:00.885 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS-9 - AppStream 15 MB/s | 6.7 MB 00:00 2024-04-05T02:46:02.344 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS-9 - BaseOS 5.6 MB/s | 2.0 MB 00:00 2024-04-05T02:46:02.424 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS-9 - BaseOS 4.9 MB/s | 2.0 MB 00:00 2024-04-05T02:46:03.298 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.0 MB 00:00 2024-04-05T02:46:05.076 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - BaseOS 3.3 MB/s | 8.0 MB 00:02 2024-04-05T02:46:07.324 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - AppStream 6.5 MB/s | 19 MB 00:02 2024-04-05T02:46:11.735 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - CRB 8.3 MB/s | 6.1 MB 00:00 2024-04-05T02:46:14.059 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - Extras packages 14 kB/s | 16 kB 00:01 2024-04-05T02:46:16.121 INFO:teuthology.orchestra.run.smithi033.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-04-05T02:46:21.194 INFO:teuthology.orchestra.run.smithi033.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-04-05T02:46:23.924 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-radosgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 14 M 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout:Upgrading: 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout: librados2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.1 M 2024-04-05T02:46:23.936 INFO:teuthology.orchestra.run.smithi033.stdout: librbd1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-base x86_64 2:17.2.7-815.g93c790e1.el9 ceph 5.0 M 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 22 M 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-selinux x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 637 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: libradosstriper1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 458 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: librgw2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 4.9 M 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-05T02:46:23.937 INFO:teuthology.orchestra.run.smithi033.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-argparse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 45 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 117 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cephfs x86_64 2:17.2.7-815.g93c790e1.el9 ceph 156 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rados x86_64 2:17.2.7-815.g93c790e1.el9 ceph 320 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 293 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 99 k 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout:Install 22 Packages 2024-04-05T02:46:23.938 INFO:teuthology.orchestra.run.smithi033.stdout:Upgrade 2 Packages 2024-04-05T02:46:23.939 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:23.939 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 58 M 2024-04-05T02:46:23.939 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:46:25.412 INFO:teuthology.orchestra.run.smithi033.stdout:(1/24): ceph-base-17.2.7-815.g93c790e1.el9.x86_ 4.4 MB/s | 5.0 MB 00:01 2024-04-05T02:46:25.496 INFO:teuthology.orchestra.run.smithi033.stdout:(2/24): ceph-selinux-17.2.7-815.g93c790e1.el9.x 301 kB/s | 25 kB 00:00 2024-04-05T02:46:25.671 INFO:teuthology.orchestra.run.smithi033.stdout:(3/24): libcephfs2-17.2.7-815.g93c790e1.el9.x86 3.6 MB/s | 637 kB 00:00 2024-04-05T02:46:25.787 INFO:teuthology.orchestra.run.smithi033.stdout:(4/24): libradosstriper1-17.2.7-815.g93c790e1.e 3.8 MB/s | 458 kB 00:00 2024-04-05T02:46:26.238 INFO:teuthology.orchestra.run.smithi033.stdout:(5/24): ceph-common-17.2.7-815.g93c790e1.el9.x8 11 MB/s | 22 MB 00:01 2024-04-05T02:46:26.388 INFO:teuthology.orchestra.run.smithi033.stdout:(6/24): ceph-radosgw-17.2.7-815.g93c790e1.el9.x 6.7 MB/s | 14 MB 00:02 2024-04-05T02:46:26.438 INFO:teuthology.orchestra.run.smithi033.stdout:(7/24): python3-ceph-argparse-17.2.7-815.g93c79 225 kB/s | 45 kB 00:00 2024-04-05T02:46:26.488 INFO:teuthology.orchestra.run.smithi033.stdout:(8/24): python3-ceph-common-17.2.7-815.g93c790e 1.1 MB/s | 117 kB 00:00 2024-04-05T02:46:26.514 INFO:teuthology.orchestra.run.smithi033.stdout:(9/24): python3-cephfs-17.2.7-815.g93c790e1.el9 2.0 MB/s | 156 kB 00:00 2024-04-05T02:46:26.598 INFO:teuthology.orchestra.run.smithi033.stdout:(10/24): librgw2-17.2.7-815.g93c790e1.el9.x86_6 6.1 MB/s | 4.9 MB 00:00 2024-04-05T02:46:26.623 INFO:teuthology.orchestra.run.smithi033.stdout:(11/24): python3-rbd-17.2.7-815.g93c790e1.el9.x 2.6 MB/s | 293 kB 00:00 2024-04-05T02:46:26.648 INFO:teuthology.orchestra.run.smithi033.stdout:(12/24): python3-rados-17.2.7-815.g93c790e1.el9 2.0 MB/s | 320 kB 00:00 2024-04-05T02:46:26.673 INFO:teuthology.orchestra.run.smithi033.stdout:(13/24): python3-rgw-17.2.7-815.g93c790e1.el9.x 1.3 MB/s | 99 kB 00:00 2024-04-05T02:46:26.807 INFO:teuthology.orchestra.run.smithi033.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 278 kB/s | 44 kB 00:00 2024-04-05T02:46:26.832 INFO:teuthology.orchestra.run.smithi033.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 762 kB/s | 159 kB 00:00 2024-04-05T02:46:26.857 INFO:teuthology.orchestra.run.smithi033.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 5.5 MB/s | 282 kB 00:00 2024-04-05T02:46:26.882 INFO:teuthology.orchestra.run.smithi033.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 826 kB/s | 41 kB 00:00 2024-04-05T02:46:26.941 INFO:teuthology.orchestra.run.smithi033.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 836 kB/s | 49 kB 00:00 2024-04-05T02:46:26.967 INFO:teuthology.orchestra.run.smithi033.stdout:(19/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.7 MB/s | 661 kB 00:00 2024-04-05T02:46:27.000 INFO:teuthology.orchestra.run.smithi033.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2024-04-05T02:46:27.025 INFO:teuthology.orchestra.run.smithi033.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.8 MB/s | 309 kB 00:00 2024-04-05T02:46:27.134 INFO:teuthology.orchestra.run.smithi033.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 9.5 MB/s | 1.6 MB 00:00 2024-04-05T02:46:27.260 INFO:teuthology.orchestra.run.smithi033.stdout:(23/24): librbd1-17.2.7-815.g93c790e1.el9.x86_6 13 MB/s | 2.9 MB 00:00 2024-04-05T02:46:27.360 INFO:teuthology.orchestra.run.smithi033.stdout:(24/24): librados2-17.2.7-815.g93c790e1.el9.x86 8.6 MB/s | 3.1 MB 00:00 2024-04-05T02:46:27.363 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:46:27.364 INFO:teuthology.orchestra.run.smithi033.stdout:Total 17 MB/s | 58 MB 00:03 2024-04-05T02:46:27.364 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:46:27.399 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:46:27.399 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:46:27.434 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - AppStream 908 kB/s | 19 MB 00:21 2024-04-05T02:46:27.876 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:46:27.877 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:46:29.154 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:46:29.507 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-05T02:46:29.562 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:46:29.637 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:46:29.718 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:46:29.747 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:46:29.818 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:46:29.873 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-05T02:46:29.901 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-05T02:46:30.357 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-05T02:46:30.407 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:46:30.470 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:46:30.500 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-05T02:46:30.543 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 10/26 2024-04-05T02:46:30.585 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:46:30.652 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:46:30.677 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:46:30.743 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:46:30.784 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-05T02:46:30.828 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-05T02:46:30.860 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-05T02:46:31.170 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-05T02:46:31.198 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:46:31.280 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:46:31.356 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 19/26 2024-04-05T02:46:31.393 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 20/26 2024-04-05T02:46:33.887 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:46:33.923 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:46:34.467 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:46:34.501 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:46:34.592 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:46:34.592 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-05T02:46:34.593 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-05T02:46:34.593 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:34.761 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:46:40.502 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - CRB 658 kB/s | 6.1 MB 00:09 2024-04-05T02:46:42.092 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 9 - Extras packages 37 kB/s | 16 kB 00:00 2024-04-05T02:46:44.766 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /sys 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /proc 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /mnt 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /var/tmp 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /home 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /root 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /tmp 2024-04-05T02:46:44.767 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:46.091 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:46:46.169 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:46:46.169 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:46:46.169 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-05T02:46:46.170 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:46:46.170 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:46:46.170 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:46.171 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:46:46.252 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:46:46.253 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:46:47.410 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:46:47.410 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 1/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 5/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 6/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 7/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 8/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 9/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 10/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-05T02:46:47.411 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-05T02:46:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 25/26 2024-04-05T02:46:48.257 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:46:48.257 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:48.257 INFO:teuthology.orchestra.run.smithi033.stdout:Upgraded: 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-05T02:46:48.258 INFO:teuthology.orchestra.run.smithi033.stdout: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:48.259 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:46:48.576 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-test 2024-04-05T02:46:49.143 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:28 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:46:49.301 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:46:49.302 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:46:49.302 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:46:49.302 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-test x86_64 2:17.2.7-815.g93c790e1.el9 ceph 48 M 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: libcephsqlite x86_64 2:17.2.7-815.g93c790e1.el9 ceph 163 k 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout:Install 6 Packages 2024-04-05T02:46:49.303 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:46:49.304 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 49 M 2024-04-05T02:46:49.304 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 193 M 2024-04-05T02:46:49.304 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:46:49.527 INFO:teuthology.orchestra.run.smithi033.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.1 MB/s | 217 kB 00:00 2024-04-05T02:46:49.577 INFO:teuthology.orchestra.run.smithi033.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 5.9 MB/s | 300 kB 00:00 2024-04-05T02:46:49.602 INFO:teuthology.orchestra.run.smithi033.stdout:(3/6): libcephsqlite-17.2.7-815.g93c790e1.el9.x 621 kB/s | 163 kB 00:00 2024-04-05T02:46:49.628 INFO:teuthology.orchestra.run.smithi033.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 63 kB 00:00 2024-04-05T02:46:49.828 INFO:teuthology.orchestra.run.smithi033.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 831 kB/s | 187 kB 00:00 2024-04-05T02:46:52.062 INFO:teuthology.orchestra.run.smithi033.stdout:(6/6): ceph-test-17.2.7-815.g93c790e1.el9.x86_6 18 MB/s | 48 MB 00:02 2024-04-05T02:46:52.065 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:46:52.065 INFO:teuthology.orchestra.run.smithi033.stdout:Total 18 MB/s | 49 MB 00:02 2024-04-05T02:46:52.079 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:46:52.101 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:46:52.101 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:46:52.503 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:46:52.504 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:46:53.101 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:46:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-05T02:46:53.228 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-05T02:46:53.255 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:46:53.298 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-05T02:46:53.316 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:46:57.947 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:46:58.026 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:47:00.251 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:47:00.251 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 1/6 2024-04-05T02:47:00.251 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2/6 2024-04-05T02:47:00.251 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:47:00.251 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-04-05T02:47:00.252 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-04-05T02:47:00.628 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-04-05T02:47:00.629 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:00.629 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:00.629 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:00.629 INFO:teuthology.orchestra.run.smithi033.stdout: jq-1.6-16.el9.x86_64 2024-04-05T02:47:00.629 INFO:teuthology.orchestra.run.smithi033.stdout: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:00.629 INFO:teuthology.orchestra.run.smithi033.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-05T02:47:00.630 INFO:teuthology.orchestra.run.smithi033.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-05T02:47:00.630 INFO:teuthology.orchestra.run.smithi033.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-05T02:47:00.630 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:00.630 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:00.914 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph 2024-04-05T02:47:01.494 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:40 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:01.663 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:01.665 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================== 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================== 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: ceph x86_64 2:17.2.7-815.g93c790e1.el9 ceph 6.5 k 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mds x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.1 M 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr x86_64 2:17.2.7-815.g93c790e1.el9 ceph 1.4 M 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-modules-core noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 240 k 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mon x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.9 M 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-osd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 15 M 2024-04-05T02:47:01.666 INFO:teuthology.orchestra.run.smithi033.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-05T02:47:01.667 INFO:teuthology.orchestra.run.smithi033.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-05T02:47:01.668 INFO:teuthology.orchestra.run.smithi033.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-05T02:47:01.669 INFO:teuthology.orchestra.run.smithi033.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-05T02:47:01.669 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:01.669 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:01.669 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================== 2024-04-05T02:47:01.669 INFO:teuthology.orchestra.run.smithi033.stdout:Install 33 Packages 2024-04-05T02:47:01.669 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:01.670 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 27 M 2024-04-05T02:47:01.670 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 96 M 2024-04-05T02:47:01.670 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:02.474 INFO:teuthology.orchestra.run.smithi033.stdout:(1/33): ceph-17.2.7-815.g93c790e1.el9.x86_64.rp 36 kB/s | 6.5 kB 00:00 2024-04-05T02:47:02.707 INFO:teuthology.orchestra.run.smithi033.stdout:(2/33): ceph-mds-17.2.7-815.g93c790e1.el9.x86_6 5.0 MB/s | 2.1 MB 00:00 2024-04-05T02:47:02.999 INFO:teuthology.orchestra.run.smithi033.stdout:(3/33): ceph-mon-17.2.7-815.g93c790e1.el9.x86_6 7.5 MB/s | 3.9 MB 00:00 2024-04-05T02:47:03.042 INFO:teuthology.orchestra.run.smithi033.stdout:(4/33): ceph-mgr-17.2.7-815.g93c790e1.el9.x86_6 1.9 MB/s | 1.4 MB 00:00 2024-04-05T02:47:03.067 INFO:teuthology.orchestra.run.smithi033.stdout:(5/33): ceph-mgr-modules-core-17.2.7-815.g93c79 3.5 MB/s | 240 kB 00:00 2024-04-05T02:47:03.200 INFO:teuthology.orchestra.run.smithi033.stdout:(6/33): python3-mako-1.1.4-6.el9.noarch.rpm 910 kB/s | 144 kB 00:00 2024-04-05T02:47:03.217 INFO:teuthology.orchestra.run.smithi033.stdout:(7/33): python3-markupsafe-1.1.1-12.el9.x86_64. 252 kB/s | 32 kB 00:00 2024-04-05T02:47:03.242 INFO:teuthology.orchestra.run.smithi033.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.1 MB/s | 47 kB 00:00 2024-04-05T02:47:03.284 INFO:teuthology.orchestra.run.smithi033.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 578 kB/s | 39 kB 00:00 2024-04-05T02:47:03.309 INFO:teuthology.orchestra.run.smithi033.stdout:(10/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 3.5 MB/s | 241 kB 00:00 2024-04-05T02:47:03.393 INFO:teuthology.orchestra.run.smithi033.stdout:(11/33): python3-cryptography-36.0.1-4.el9.x86_ 11 MB/s | 1.1 MB 00:00 2024-04-05T02:47:03.418 INFO:teuthology.orchestra.run.smithi033.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 950 kB/s | 103 kB 00:00 2024-04-05T02:47:03.443 INFO:teuthology.orchestra.run.smithi033.stdout:(13/33): python3-pycparser-2.20-6.el9.noarch.rp 2.4 MB/s | 124 kB 00:00 2024-04-05T02:47:03.484 INFO:teuthology.orchestra.run.smithi033.stdout:(14/33): python3-urllib3-1.26.5-5.el9.noarch.rp 2.8 MB/s | 187 kB 00:00 2024-04-05T02:47:03.644 INFO:teuthology.orchestra.run.smithi033.stdout:(15/33): ceph-osd-17.2.7-815.g93c790e1.el9.x86_ 16 MB/s | 15 MB 00:00 2024-04-05T02:47:03.703 INFO:teuthology.orchestra.run.smithi033.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 199 kB/s | 43 kB 00:00 2024-04-05T02:47:03.770 INFO:teuthology.orchestra.run.smithi033.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-04-05T02:47:03.803 INFO:teuthology.orchestra.run.smithi033.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 321 kB/s | 11 kB 00:00 2024-04-05T02:47:03.854 INFO:teuthology.orchestra.run.smithi033.stdout:(19/33): python3-cherrypy-18.6.1-2.el9.noarch.r 2.3 MB/s | 358 kB 00:00 2024-04-05T02:47:03.879 INFO:teuthology.orchestra.run.smithi033.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 359 kB/s | 18 kB 00:00 2024-04-05T02:47:03.904 INFO:teuthology.orchestra.run.smithi033.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 462 kB/s | 23 kB 00:00 2024-04-05T02:47:03.929 INFO:teuthology.orchestra.run.smithi033.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-04-05T02:47:03.955 INFO:teuthology.orchestra.run.smithi033.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 391 kB/s | 20 kB 00:00 2024-04-05T02:47:03.988 INFO:teuthology.orchestra.run.smithi033.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 802 kB/s | 46 kB 00:00 2024-04-05T02:47:04.014 INFO:teuthology.orchestra.run.smithi033.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-04-05T02:47:04.056 INFO:teuthology.orchestra.run.smithi033.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 396 kB/s | 16 kB 00:00 2024-04-05T02:47:04.081 INFO:teuthology.orchestra.run.smithi033.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.9 MB/s | 272 kB 00:00 2024-04-05T02:47:04.106 INFO:teuthology.orchestra.run.smithi033.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-04-05T02:47:04.131 INFO:teuthology.orchestra.run.smithi033.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 713 kB/s | 36 kB 00:00 2024-04-05T02:47:04.165 INFO:teuthology.orchestra.run.smithi033.stdout:(30/33): python3-webob-1.8.7-6.el9.noarch.rpm 3.8 MB/s | 230 kB 00:00 2024-04-05T02:47:04.207 INFO:teuthology.orchestra.run.smithi033.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 482 kB/s | 20 kB 00:00 2024-04-05T02:47:04.232 INFO:teuthology.orchestra.run.smithi033.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.2 MB/s | 427 kB 00:00 2024-04-05T02:47:04.391 INFO:teuthology.orchestra.run.smithi033.stdout:(33/33): python3-requests-2.25.1-8.el9.noarch.r 132 kB/s | 125 kB 00:00 2024-04-05T02:47:04.396 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:04.396 INFO:teuthology.orchestra.run.smithi033.stdout:Total 9.9 MB/s | 27 MB 00:02 2024-04-05T02:47:04.410 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:04.453 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:04.453 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:04.815 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:04.815 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:06.061 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:06.135 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-05T02:47:06.168 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-05T02:47:06.200 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-05T02:47:06.297 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-05T02:47:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-05T02:47:06.428 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-05T02:47:06.503 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-05T02:47:06.555 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-04-05T02:47:06.593 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-04-05T02:47:06.651 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-04-05T02:47:06.714 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-04-05T02:47:06.917 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-04-05T02:47:06.971 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-04-05T02:47:07.010 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-04-05T02:47:07.045 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-04-05T02:47:07.074 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-05T02:47:07.154 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-04-05T02:47:07.193 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:47:07.226 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-04-05T02:47:07.259 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-04-05T02:47:07.302 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-04-05T02:47:07.328 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-04-05T02:47:07.455 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-04-05T02:47:07.492 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-04-05T02:47:07.576 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-04-05T02:47:07.726 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-04-05T02:47:07.855 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:47:08.005 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 28/33 2024-04-05T02:47:08.030 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:47:08.055 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:47:08.055 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:47:08.055 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-05T02:47:08.056 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:47:08.056 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:47:08.056 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:09.448 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:47:09.474 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:47:09.474 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:47:09.474 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-05T02:47:09.474 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:47:09.474 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:47:09.474 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:09.865 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:47:09.888 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:47:09.888 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:47:09.888 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-05T02:47:09.888 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:47:09.888 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:47:09.888 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:10.099 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:47:10.122 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:47:10.122 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:47:10.122 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-05T02:47:10.122 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:47:10.122 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:47:10.122 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:10.190 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:47:11.807 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:47:11.807 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 1/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 3/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 4/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 5/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 6/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-04-05T02:47:11.808 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-05T02:47:11.809 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-05T02:47:11.810 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-05T02:47:11.810 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-05T02:47:12.693 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-05T02:47:12.694 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ply-3.11-14.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-05T02:47:12.695 INFO:teuthology.orchestra.run.smithi033.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-05T02:47:12.696 INFO:teuthology.orchestra.run.smithi033.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-05T02:47:12.696 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:12.696 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:13.062 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-base 2024-04-05T02:47:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:52 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:13.737 INFO:teuthology.orchestra.run.smithi033.stdout:Package ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:47:13.791 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:13.792 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:47:13.792 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:13.853 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install cephadm 2024-04-05T02:47:14.425 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:53 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:14.582 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:14.583 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:14.583 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:47:14.583 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:14.583 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:14.583 INFO:teuthology.orchestra.run.smithi033.stdout: cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 82 k 2024-04-05T02:47:14.583 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:14.583 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:14.584 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:14.584 INFO:teuthology.orchestra.run.smithi033.stdout:Install 1 Package 2024-04-05T02:47:14.584 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:14.584 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 82 k 2024-04-05T02:47:14.584 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 372 k 2024-04-05T02:47:14.584 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:14.981 INFO:teuthology.orchestra.run.smithi033.stdout:cephadm-17.2.7-815.g93c790e1.el9.noarch.rpm 206 kB/s | 82 kB 00:00 2024-04-05T02:47:14.982 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:14.982 INFO:teuthology.orchestra.run.smithi033.stdout:Total 206 kB/s | 82 kB 00:00 2024-04-05T02:47:14.982 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:14.988 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:14.988 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:14.990 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:14.990 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:15.065 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:15.514 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:47:15.622 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:47:15.969 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:47:16.234 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:47:16.235 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:16.235 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:16.235 INFO:teuthology.orchestra.run.smithi033.stdout: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:16.235 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:16.235 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:16.431 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-immutable-object-cache 2024-04-05T02:47:17.004 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:56 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:17.164 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repo Size 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-815.g93c790e1.el9 ceph 137 k 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout:Install 1 Package 2024-04-05T02:47:17.165 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:17.166 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 137 k 2024-04-05T02:47:17.166 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 379 k 2024-04-05T02:47:17.166 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:17.721 INFO:teuthology.orchestra.run.smithi033.stdout:ceph-immutable-object-cache-17.2.7-815.g93c790e 248 kB/s | 137 kB 00:00 2024-04-05T02:47:17.722 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:17.722 INFO:teuthology.orchestra.run.smithi033.stdout:Total 247 kB/s | 137 kB 00:00 2024-04-05T02:47:17.722 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:17.730 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:17.730 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:17.785 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:17.786 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:17.926 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:17.988 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:47:18.014 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:47:18.014 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:47:18.014 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-05T02:47:18.014 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:19.021 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:47:19.021 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:19.021 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:19.021 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:19.021 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:19.021 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:19.274 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-mgr 2024-04-05T02:47:19.849 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:58 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:19.950 INFO:teuthology.orchestra.run.smithi033.stdout:Package ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:47:20.005 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:20.006 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:47:20.006 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:20.069 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-mgr-dashboard 2024-04-05T02:47:20.637 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:59 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:20.793 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:20.793 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:20.793 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-dashboard noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 1.7 M 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-grafana-dashboards noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 24 k 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-prometheus-alerts noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 15 k 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout:Installing weak dependencies: 2024-04-05T02:47:20.794 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-05T02:47:20.795 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:20.795 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:20.795 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:20.795 INFO:teuthology.orchestra.run.smithi033.stdout:Install 7 Packages 2024-04-05T02:47:20.795 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:20.795 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 2.0 M 2024-04-05T02:47:20.795 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 8.1 M 2024-04-05T02:47:20.796 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:21.424 INFO:teuthology.orchestra.run.smithi033.stdout:(1/7): ceph-prometheus-alerts-17.2.7-815.g93c79 33 kB/s | 15 kB 00:00 2024-04-05T02:47:21.475 INFO:teuthology.orchestra.run.smithi033.stdout:(2/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 181 kB/s | 9.0 kB 00:00 2024-04-05T02:47:21.500 INFO:teuthology.orchestra.run.smithi033.stdout:(3/7): ceph-grafana-dashboards-17.2.7-815.g93c7 46 kB/s | 24 kB 00:00 2024-04-05T02:47:21.533 INFO:teuthology.orchestra.run.smithi033.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 697 kB/s | 41 kB 00:00 2024-04-05T02:47:21.592 INFO:teuthology.orchestra.run.smithi033.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 334 kB/s | 31 kB 00:00 2024-04-05T02:47:21.626 INFO:teuthology.orchestra.run.smithi033.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 2.0 MB/s | 188 kB 00:00 2024-04-05T02:47:22.909 INFO:teuthology.orchestra.run.smithi033.stdout:(7/7): ceph-mgr-dashboard-17.2.7-815.g93c790e1. 902 kB/s | 1.7 MB 00:01 2024-04-05T02:47:22.911 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:22.911 INFO:teuthology.orchestra.run.smithi033.stdout:Total 965 kB/s | 2.0 MB 00:02 2024-04-05T02:47:22.912 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:22.921 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:22.921 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:22.993 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:22.994 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:23.297 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:23.352 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-05T02:47:23.386 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-05T02:47:23.433 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-05T02:47:23.473 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-05T02:47:23.502 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 5/7 2024-04-05T02:47:23.740 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 6/7 2024-04-05T02:47:23.771 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:47:24.311 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:47:24.311 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 1/7 2024-04-05T02:47:24.312 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2/7 2024-04-05T02:47:24.312 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 3/7 2024-04-05T02:47:24.312 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-05T02:47:24.312 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-05T02:47:24.312 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-05T02:47:24.598 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-05T02:47:24.599 INFO:teuthology.orchestra.run.smithi033.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-05T02:47:24.599 INFO:teuthology.orchestra.run.smithi033.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-05T02:47:24.599 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:24.599 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:24.825 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-05T02:47:25.399 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:04 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:25.559 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout:=========================================================================================== 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout:=========================================================================================== 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 7.4 M 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-04-05T02:47:25.560 INFO:teuthology.orchestra.run.smithi033.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout:=========================================================================================== 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout:Install 12 Packages 2024-04-05T02:47:25.561 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:25.562 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 37 M 2024-04-05T02:47:25.562 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 209 M 2024-04-05T02:47:25.562 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:26.225 INFO:teuthology.orchestra.run.smithi033.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 238 kB/s | 32 kB 00:00 2024-04-05T02:47:26.250 INFO:teuthology.orchestra.run.smithi033.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 726 kB/s | 17 kB 00:00 2024-04-05T02:47:26.283 INFO:teuthology.orchestra.run.smithi033.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 1.0 MB/s | 34 kB 00:00 2024-04-05T02:47:26.334 INFO:teuthology.orchestra.run.smithi033.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 12 MB/s | 3.0 MB 00:00 2024-04-05T02:47:26.642 INFO:teuthology.orchestra.run.smithi033.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 13 MB/s | 4.7 MB 00:00 2024-04-05T02:47:26.726 INFO:teuthology.orchestra.run.smithi033.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 12 MB/s | 4.8 MB 00:00 2024-04-05T02:47:26.743 INFO:teuthology.orchestra.run.smithi033.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.2 MB/s | 217 kB 00:00 2024-04-05T02:47:26.968 INFO:teuthology.orchestra.run.smithi033.stdout:(8/12): libgfortran-11.4.1-3.el9.x86_64.rpm 3.5 MB/s | 803 kB 00:00 2024-04-05T02:47:27.010 INFO:teuthology.orchestra.run.smithi033.stdout:(9/12): libquadmath-11.4.1-3.el9.x86_64.rpm 4.8 MB/s | 195 kB 00:00 2024-04-05T02:47:27.335 INFO:teuthology.orchestra.run.smithi033.stdout:(10/12): python3-devel-3.9.17-2.el9.x86_64.rpm 756 kB/s | 245 kB 00:00 2024-04-05T02:47:27.494 INFO:teuthology.orchestra.run.smithi033.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 20 MB/s | 16 MB 00:00 2024-04-05T02:47:29.894 INFO:teuthology.orchestra.run.smithi033.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-8 1.9 MB/s | 7.4 MB 00:03 2024-04-05T02:47:29.896 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:29.896 INFO:teuthology.orchestra.run.smithi033.stdout:Total 8.6 MB/s | 37 MB 00:04 2024-04-05T02:47:29.941 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:29.967 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:29.967 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:30.418 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:30.419 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:31.250 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:31.325 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-04-05T02:47:31.356 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-05T02:47:31.455 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:47:31.493 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-05T02:47:31.676 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:47:31.709 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:47:31.793 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-05T02:47:32.474 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-05T02:47:32.582 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-05T02:47:33.886 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-05T02:47:34.965 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-05T02:47:34.992 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:47:37.830 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:47:37.830 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 1/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-04-05T02:47:37.831 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-04-05T02:47:38.214 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-04-05T02:47:38.214 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:38.214 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:38.214 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:38.214 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-05T02:47:38.214 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:38.215 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:38.571 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-mgr-rook 2024-04-05T02:47:39.153 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:18 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:39.314 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-rook noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 47 k 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-04-05T02:47:39.316 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout:Install 13 Packages 2024-04-05T02:47:39.317 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:39.318 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 2.0 M 2024-04-05T02:47:39.318 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 26 M 2024-04-05T02:47:39.318 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:39.516 INFO:teuthology.orchestra.run.smithi033.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 160 kB/s | 18 kB 00:00 2024-04-05T02:47:39.541 INFO:teuthology.orchestra.run.smithi033.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 177 kB/s | 24 kB 00:00 2024-04-05T02:47:39.591 INFO:teuthology.orchestra.run.smithi033.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.3 MB/s | 174 kB 00:00 2024-04-05T02:47:39.608 INFO:teuthology.orchestra.run.smithi033.stdout:(4/13): ceph-mgr-rook-17.2.7-815.g93c790e1.el9. 231 kB/s | 47 kB 00:00 2024-04-05T02:47:39.633 INFO:teuthology.orchestra.run.smithi033.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.4 MB/s | 132 kB 00:00 2024-04-05T02:47:39.659 INFO:teuthology.orchestra.run.smithi033.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 3.1 MB/s | 211 kB 00:00 2024-04-05T02:47:39.684 INFO:teuthology.orchestra.run.smithi033.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 855 kB/s | 43 kB 00:00 2024-04-05T02:47:39.826 INFO:teuthology.orchestra.run.smithi033.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 90 kB/s | 15 kB 00:00 2024-04-05T02:47:39.851 INFO:teuthology.orchestra.run.smithi033.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 148 kB/s | 32 kB 00:00 2024-04-05T02:47:39.968 INFO:teuthology.orchestra.run.smithi033.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 506 kB/s | 59 kB 00:00 2024-04-05T02:47:39.994 INFO:teuthology.orchestra.run.smithi033.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 656 kB/s | 203 kB 00:00 2024-04-05T02:47:40.119 INFO:teuthology.orchestra.run.smithi033.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 597 kB/s | 90 kB 00:00 2024-04-05T02:47:40.311 INFO:teuthology.orchestra.run.smithi033.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 2.1 MB/s | 1.0 MB 00:00 2024-04-05T02:47:40.313 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:40.313 INFO:teuthology.orchestra.run.smithi033.stdout:Total 2.1 MB/s | 2.0 MB 00:00 2024-04-05T02:47:40.313 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:40.328 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:40.328 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:40.429 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:40.429 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:40.951 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:41.134 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-05T02:47:41.189 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-05T02:47:41.243 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-05T02:47:41.280 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-05T02:47:41.322 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-05T02:47:41.419 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-05T02:47:41.528 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-05T02:47:41.582 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-05T02:47:42.343 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-05T02:47:42.394 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-05T02:47:42.429 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-05T02:47:42.475 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-05T02:47:42.498 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:47:43.257 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 1/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-05T02:47:43.258 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-05T02:47:43.612 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-05T02:47:43.612 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:43.612 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:43.612 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:43.612 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-05T02:47:43.612 INFO:teuthology.orchestra.run.smithi033.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-05T02:47:43.612 INFO:teuthology.orchestra.run.smithi033.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:43.613 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:43.863 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-mgr-cephadm 2024-04-05T02:47:44.437 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:23 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:44.596 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:44.596 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-04-05T02:47:44.596 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 123 k 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-04-05T02:47:44.597 INFO:teuthology.orchestra.run.smithi033.stdout:Install 6 Packages 2024-04-05T02:47:44.598 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:44.598 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 6.8 M 2024-04-05T02:47:44.598 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 33 M 2024-04-05T02:47:44.598 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:45.395 INFO:teuthology.orchestra.run.smithi033.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.5 MB/s | 248 kB 00:00 2024-04-05T02:47:45.762 INFO:teuthology.orchestra.run.smithi033.stdout:(2/6): ceph-mgr-cephadm-17.2.7-815.g93c790e1.el 235 kB/s | 123 kB 00:00 2024-04-05T02:47:46.021 INFO:teuthology.orchestra.run.smithi033.stdout:(3/6): python3-babel-2.9.1-2.el9.noarch.rpm 7.4 MB/s | 5.8 MB 00:00 2024-04-05T02:47:46.037 INFO:teuthology.orchestra.run.smithi033.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 210 kB/s | 58 kB 00:00 2024-04-05T02:47:46.071 INFO:teuthology.orchestra.run.smithi033.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 811 kB/s | 548 kB 00:00 2024-04-05T02:47:46.096 INFO:teuthology.orchestra.run.smithi033.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 677 kB/s | 51 kB 00:00 2024-04-05T02:47:46.099 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:46.099 INFO:teuthology.orchestra.run.smithi033.stdout:Total 4.5 MB/s | 6.8 MB 00:01 2024-04-05T02:47:46.113 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:46.124 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:46.124 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:46.215 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:46.216 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:46.537 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:46.713 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-05T02:47:46.766 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-05T02:47:47.319 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-05T02:47:47.412 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-05T02:47:47.499 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-05T02:47:47.527 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:47:48.304 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:47:48.304 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/6 2024-04-05T02:47:48.304 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-05T02:47:48.304 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-05T02:47:48.304 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-05T02:47:48.304 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-05T02:47:48.610 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:48.611 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:48.838 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-fuse 2024-04-05T02:47:49.415 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:28 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 812 k 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:49.573 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:49.574 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:49.574 INFO:teuthology.orchestra.run.smithi033.stdout:Install 2 Packages 2024-04-05T02:47:49.574 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:49.574 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 891 k 2024-04-05T02:47:49.574 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 2.6 M 2024-04-05T02:47:49.574 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:49.739 INFO:teuthology.orchestra.run.smithi033.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 475 kB/s | 79 kB 00:00 2024-04-05T02:47:50.023 INFO:teuthology.orchestra.run.smithi033.stdout:(2/2): ceph-fuse-17.2.7-815.g93c790e1.el9.x86_6 1.8 MB/s | 812 kB 00:00 2024-04-05T02:47:50.024 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:50.024 INFO:teuthology.orchestra.run.smithi033.stdout:Total 1.9 MB/s | 891 kB 00:00 2024-04-05T02:47:50.024 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:50.046 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:50.046 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:50.121 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:50.121 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:50.440 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:50.580 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-05T02:47:50.650 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:47:51.338 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:47:51.338 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/2 2024-04-05T02:47:51.676 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-05T02:47:51.676 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:51.676 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:51.676 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-05T02:47:51.677 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:51.677 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:51.910 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install librados-devel 2024-04-05T02:47:52.488 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:31 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:52.645 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repo Size 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout: librados-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 126 k 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:52.646 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:52.647 INFO:teuthology.orchestra.run.smithi033.stdout:Install 1 Package 2024-04-05T02:47:52.647 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:52.647 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 126 k 2024-04-05T02:47:52.647 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 448 k 2024-04-05T02:47:52.647 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:52.960 INFO:teuthology.orchestra.run.smithi033.stdout:librados-devel-17.2.7-815.g93c790e1.el9.x86_64. 403 kB/s | 126 kB 00:00 2024-04-05T02:47:52.961 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:52.961 INFO:teuthology.orchestra.run.smithi033.stdout:Total 401 kB/s | 126 kB 00:00 2024-04-05T02:47:52.961 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:52.967 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:52.967 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:53.025 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:53.025 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:53.286 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:47:53.719 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:47:53.984 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:47:53.985 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:53.985 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:53.985 INFO:teuthology.orchestra.run.smithi033.stdout: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:53.985 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:53.985 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:54.254 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install libcephfs2 2024-04-05T02:47:54.828 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:33 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:54.929 INFO:teuthology.orchestra.run.smithi033.stdout:Package libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:47:54.985 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:54.986 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:47:54.986 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:55.047 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install libcephfs-devel 2024-04-05T02:47:55.637 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:34 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:55.808 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repo Size 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout:Install 1 Package 2024-04-05T02:47:55.809 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:55.810 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 25 k 2024-04-05T02:47:55.810 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 110 k 2024-04-05T02:47:55.810 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:47:56.007 INFO:teuthology.orchestra.run.smithi033.stdout:libcephfs-devel-17.2.7-815.g93c790e1.el9.x86_64 127 kB/s | 25 kB 00:00 2024-04-05T02:47:56.008 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:47:56.008 INFO:teuthology.orchestra.run.smithi033.stdout:Total 126 kB/s | 25 kB 00:00 2024-04-05T02:47:56.008 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:47:56.011 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:47:56.011 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:47:56.013 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:47:56.013 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:47:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:47:56.216 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:47:56.564 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:47:56.825 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:47:56.825 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:56.825 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:47:56.825 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:47:56.825 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:47:56.825 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:56.997 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install librados2 2024-04-05T02:47:57.572 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:36 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:57.673 INFO:teuthology.orchestra.run.smithi033.stdout:Package librados2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:47:57.728 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:57.729 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:47:57.729 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:57.789 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install librbd1 2024-04-05T02:47:58.361 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:37 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:58.463 INFO:teuthology.orchestra.run.smithi033.stdout:Package librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:47:58.521 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:58.522 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:47:58.522 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:58.580 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install python3-rados 2024-04-05T02:47:59.155 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:38 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:47:59.257 INFO:teuthology.orchestra.run.smithi033.stdout:Package python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:47:59.313 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:47:59.314 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:47:59.314 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:47:59.372 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install python3-rgw 2024-04-05T02:47:59.949 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:38 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:48:00.051 INFO:teuthology.orchestra.run.smithi033.stdout:Package python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:48:00.106 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:48:00.107 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:48:00.107 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:48:00.175 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install python3-cephfs 2024-04-05T02:48:00.747 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:39 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:48:00.849 INFO:teuthology.orchestra.run.smithi033.stdout:Package python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:48:00.904 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:48:00.905 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:48:00.905 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:48:00.983 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install python3-rbd 2024-04-05T02:48:01.558 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:40 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:48:01.660 INFO:teuthology.orchestra.run.smithi033.stdout:Package python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:48:01.716 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:48:01.717 INFO:teuthology.orchestra.run.smithi033.stdout:Nothing to do. 2024-04-05T02:48:01.717 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:48:01.789 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install rbd-fuse 2024-04-05T02:48:02.365 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:41 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:48:02.522 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout: Package Architecture Version Repository Size 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 82 k 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:02.523 INFO:teuthology.orchestra.run.smithi033.stdout:Install 1 Package 2024-04-05T02:48:02.524 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:02.524 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 82 k 2024-04-05T02:48:02.524 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 222 k 2024-04-05T02:48:02.524 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:48:03.068 INFO:teuthology.orchestra.run.smithi033.stdout:rbd-fuse-17.2.7-815.g93c790e1.el9.x86_64.rpm 150 kB/s | 82 kB 00:00 2024-04-05T02:48:03.069 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:48:03.069 INFO:teuthology.orchestra.run.smithi033.stdout:Total 150 kB/s | 82 kB 00:00 2024-04-05T02:48:03.069 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:48:03.076 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:48:03.076 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:48:03.136 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:48:03.136 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:48:03.286 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:48:03.394 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:03.776 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:04.083 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:04.084 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:04.084 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:48:04.084 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:48:04.084 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:04.084 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:48:04.337 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install rbd-mirror 2024-04-05T02:48:04.914 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:43 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:48:05.073 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:48:05.073 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:05.073 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repo Size 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-mirror x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:Install 1 Package 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 2.9 M 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 12 M 2024-04-05T02:48:05.074 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:48:05.662 INFO:teuthology.orchestra.run.smithi033.stdout:rbd-mirror-17.2.7-815.g93c790e1.el9.x86_64.rpm 5.0 MB/s | 2.9 MB 00:00 2024-04-05T02:48:05.663 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:48:05.663 INFO:teuthology.orchestra.run.smithi033.stdout:Total 5.0 MB/s | 2.9 MB 00:00 2024-04-05T02:48:05.663 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:48:05.673 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:48:05.673 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:48:05.748 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:48:05.749 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:48:06.180 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:48:06.238 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:06.262 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:06.263 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:48:06.263 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-05T02:48:06.263 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:48:06.263 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:48:06.263 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:07.489 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:07.489 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:07.489 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:48:07.489 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:48:07.489 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:07.489 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:48:07.711 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install rbd-nbd 2024-04-05T02:48:08.286 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:01:47 ago on Fri 05 Apr 2024 02:46:21 AM UTC. 2024-04-05T02:48:08.442 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-04-05T02:48:08.443 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:08.443 INFO:teuthology.orchestra.run.smithi033.stdout: Package Architecture Version Repository Size 2024-04-05T02:48:08.443 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:08.443 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-04-05T02:48:08.443 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-nbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 165 k 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout:Install 1 Package 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 165 k 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 477 k 2024-04-05T02:48:08.444 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-04-05T02:48:08.883 INFO:teuthology.orchestra.run.smithi033.stdout:rbd-nbd-17.2.7-815.g93c790e1.el9.x86_64.rpm 376 kB/s | 165 kB 00:00 2024-04-05T02:48:08.884 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:48:08.884 INFO:teuthology.orchestra.run.smithi033.stdout:Total 375 kB/s | 165 kB 00:00 2024-04-05T02:48:08.884 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-04-05T02:48:08.892 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-04-05T02:48:08.892 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-04-05T02:48:08.951 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-04-05T02:48:08.951 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-04-05T02:48:09.105 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-04-05T02:48:09.200 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:09.632 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:09.923 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:48:09.923 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:09.923 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-04-05T02:48:09.923 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:48:09.924 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:48:09.924 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-04-05T02:48:10.163 DEBUG:teuthology.parallel:result is None 2024-04-05T02:53:37.064 INFO:teuthology.orchestra.run.smithi151.stdout:Extra Packages for Enterprise Linux 51 kB/s | 21 MB 06:54 2024-04-05T02:53:41.963 INFO:teuthology.orchestra.run.smithi151.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-04-05T02:53:44.556 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:53:44.568 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:53:44.568 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:53:44.568 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-radosgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 14 M 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout:Upgrading: 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: librados2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.1 M 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: librbd1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-base x86_64 2:17.2.7-815.g93c790e1.el9 ceph 5.0 M 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 22 M 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-selinux x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 637 k 2024-04-05T02:53:44.569 INFO:teuthology.orchestra.run.smithi151.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: libradosstriper1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 458 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: librgw2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 4.9 M 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-argparse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 45 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 117 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cephfs x86_64 2:17.2.7-815.g93c790e1.el9 ceph 156 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rados x86_64 2:17.2.7-815.g93c790e1.el9 ceph 320 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 293 k 2024-04-05T02:53:44.570 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 99 k 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout:Install 22 Packages 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout:Upgrade 2 Packages 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 58 M 2024-04-05T02:53:44.571 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:53:46.114 INFO:teuthology.orchestra.run.smithi151.stdout:(1/24): ceph-base-17.2.7-815.g93c790e1.el9.x86_ 3.3 MB/s | 5.0 MB 00:01 2024-04-05T02:53:46.148 INFO:teuthology.orchestra.run.smithi151.stdout:(2/24): ceph-selinux-17.2.7-815.g93c790e1.el9.x 756 kB/s | 25 kB 00:00 2024-04-05T02:53:46.248 INFO:teuthology.orchestra.run.smithi151.stdout:(3/24): libcephfs2-17.2.7-815.g93c790e1.el9.x86 6.2 MB/s | 637 kB 00:00 2024-04-05T02:53:46.332 INFO:teuthology.orchestra.run.smithi151.stdout:(4/24): libradosstriper1-17.2.7-815.g93c790e1.e 5.4 MB/s | 458 kB 00:00 2024-04-05T02:53:46.849 INFO:teuthology.orchestra.run.smithi151.stdout:(5/24): librgw2-17.2.7-815.g93c790e1.el9.x86_64 9.5 MB/s | 4.9 MB 00:00 2024-04-05T02:53:46.916 INFO:teuthology.orchestra.run.smithi151.stdout:(6/24): python3-ceph-argparse-17.2.7-815.g93c79 678 kB/s | 45 kB 00:00 2024-04-05T02:53:46.982 INFO:teuthology.orchestra.run.smithi151.stdout:(7/24): python3-ceph-common-17.2.7-815.g93c790e 1.7 MB/s | 117 kB 00:00 2024-04-05T02:53:47.050 INFO:teuthology.orchestra.run.smithi151.stdout:(8/24): python3-cephfs-17.2.7-815.g93c790e1.el9 2.3 MB/s | 156 kB 00:00 2024-04-05T02:53:47.124 INFO:teuthology.orchestra.run.smithi151.stdout:(9/24): python3-rados-17.2.7-815.g93c790e1.el9. 4.2 MB/s | 320 kB 00:00 2024-04-05T02:53:47.342 INFO:teuthology.orchestra.run.smithi151.stdout:(10/24): ceph-common-17.2.7-815.g93c790e1.el9.x 8.3 MB/s | 22 MB 00:02 2024-04-05T02:53:47.367 INFO:teuthology.orchestra.run.smithi151.stdout:(11/24): python3-rbd-17.2.7-815.g93c790e1.el9.x 1.2 MB/s | 293 kB 00:00 2024-04-05T02:53:47.392 INFO:teuthology.orchestra.run.smithi151.stdout:(12/24): python3-rgw-17.2.7-815.g93c790e1.el9.x 1.9 MB/s | 99 kB 00:00 2024-04-05T02:53:47.509 INFO:teuthology.orchestra.run.smithi151.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 379 kB/s | 44 kB 00:00 2024-04-05T02:53:47.534 INFO:teuthology.orchestra.run.smithi151.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 953 kB/s | 159 kB 00:00 2024-04-05T02:53:47.576 INFO:teuthology.orchestra.run.smithi151.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 9.7 MB/s | 661 kB 00:00 2024-04-05T02:53:47.610 INFO:teuthology.orchestra.run.smithi151.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.7 MB/s | 282 kB 00:00 2024-04-05T02:53:47.635 INFO:teuthology.orchestra.run.smithi151.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 708 kB/s | 41 kB 00:00 2024-04-05T02:53:47.885 INFO:teuthology.orchestra.run.smithi151.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 194 kB/s | 49 kB 00:00 2024-04-05T02:53:47.952 INFO:teuthology.orchestra.run.smithi151.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 903 kB/s | 309 kB 00:00 2024-04-05T02:53:47.977 INFO:teuthology.orchestra.run.smithi151.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 735 kB/s | 67 kB 00:00 2024-04-05T02:53:48.220 INFO:teuthology.orchestra.run.smithi151.stdout:(21/24): thrift-0.15.0-2.el9.x86_64.rpm 5.9 MB/s | 1.6 MB 00:00 2024-04-05T02:53:48.478 INFO:teuthology.orchestra.run.smithi151.stdout:(22/24): ceph-radosgw-17.2.7-815.g93c790e1.el9. 3.7 MB/s | 14 MB 00:03 2024-04-05T02:53:48.536 INFO:teuthology.orchestra.run.smithi151.stdout:(23/24): librados2-17.2.7-815.g93c790e1.el9.x86 5.5 MB/s | 3.1 MB 00:00 2024-04-05T02:53:48.778 INFO:teuthology.orchestra.run.smithi151.stdout:(24/24): librbd1-17.2.7-815.g93c790e1.el9.x86_6 5.3 MB/s | 2.9 MB 00:00 2024-04-05T02:53:48.784 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:53:48.784 INFO:teuthology.orchestra.run.smithi151.stdout:Total 14 MB/s | 58 MB 00:04 2024-04-05T02:53:48.784 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:53:48.814 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:53:48.814 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:53:49.275 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:53:49.276 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:53:50.542 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:53:50.864 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-05T02:53:50.922 INFO:teuthology.orchestra.run.smithi151.stdout: Upgrading : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:53:51.013 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:53:51.092 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:53:51.115 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:53:51.194 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:53:51.252 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-05T02:53:51.295 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-05T02:53:51.726 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-05T02:53:51.776 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:53:51.821 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:53:51.851 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-05T02:53:51.893 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 10/26 2024-04-05T02:53:51.937 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:53:52.001 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:53:52.029 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:53:52.086 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:53:52.126 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-05T02:53:52.161 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-05T02:53:52.195 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-05T02:53:52.489 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-05T02:53:52.516 INFO:teuthology.orchestra.run.smithi151.stdout: Upgrading : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:53:52.595 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:53:52.671 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 19/26 2024-04-05T02:53:52.701 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 20/26 2024-04-05T02:53:55.134 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:53:55.172 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:53:55.716 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:53:55.750 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:53:55.841 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:53:55.841 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-05T02:53:55.842 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-05T02:53:55.842 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:53:55.877 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /sys 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /proc 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /mnt 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /var/tmp 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /home 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /root 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout:skipping the directory /tmp 2024-04-05T02:54:05.398 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:06.667 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:54:06.747 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:54:06.747 INFO:teuthology.orchestra.run.smithi151.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:54:06.747 INFO:teuthology.orchestra.run.smithi151.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-05T02:54:06.747 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:54:06.747 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:54:06.748 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:06.749 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:54:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:54:06.831 INFO:teuthology.orchestra.run.smithi151.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 1/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 5/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 6/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 7/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 8/26 2024-04-05T02:54:08.129 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 9/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 10/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-05T02:54:08.130 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-05T02:54:08.131 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-05T02:54:08.131 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:54:08.131 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-05T02:54:08.131 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 25/26 2024-04-05T02:54:09.019 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:54:09.019 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:09.019 INFO:teuthology.orchestra.run.smithi151.stdout:Upgraded: 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.020 INFO:teuthology.orchestra.run.smithi151.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:09.021 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:09.375 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-test 2024-04-05T02:54:09.952 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:28 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:10.104 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:10.105 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:10.105 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:54:10.105 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:10.105 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:54:10.105 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-test x86_64 2:17.2.7-815.g93c790e1.el9 ceph 48 M 2024-04-05T02:54:10.105 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout: libcephsqlite x86_64 2:17.2.7-815.g93c790e1.el9 ceph 163 k 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout:Install 6 Packages 2024-04-05T02:54:10.106 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:10.107 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 49 M 2024-04-05T02:54:10.107 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 193 M 2024-04-05T02:54:10.107 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:54:10.335 INFO:teuthology.orchestra.run.smithi151.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.3 MB/s | 217 kB 00:00 2024-04-05T02:54:10.376 INFO:teuthology.orchestra.run.smithi151.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 7.1 MB/s | 300 kB 00:00 2024-04-05T02:54:10.410 INFO:teuthology.orchestra.run.smithi151.stdout:(3/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.8 MB/s | 63 kB 00:00 2024-04-05T02:54:10.536 INFO:teuthology.orchestra.run.smithi151.stdout:(4/6): jq-1.6-16.el9.x86_64.rpm 1.5 MB/s | 187 kB 00:00 2024-04-05T02:54:10.561 INFO:teuthology.orchestra.run.smithi151.stdout:(5/6): libcephsqlite-17.2.7-815.g93c790e1.el9.x 418 kB/s | 163 kB 00:00 2024-04-05T02:54:12.811 INFO:teuthology.orchestra.run.smithi151.stdout:(6/6): ceph-test-17.2.7-815.g93c790e1.el9.x86_6 18 MB/s | 48 MB 00:02 2024-04-05T02:54:12.814 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:54:12.814 INFO:teuthology.orchestra.run.smithi151.stdout:Total 18 MB/s | 49 MB 00:02 2024-04-05T02:54:12.829 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:54:12.849 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:54:12.849 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:54:13.229 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:54:13.229 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:54:13.808 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:54:13.889 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-05T02:54:13.933 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-05T02:54:13.970 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:54:14.012 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-05T02:54:14.040 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:54:18.462 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:54:18.532 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:54:20.973 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:54:20.973 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 1/6 2024-04-05T02:54:20.973 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2/6 2024-04-05T02:54:20.973 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:54:20.973 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-04-05T02:54:20.973 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: jq-1.6-16.el9.x86_64 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-05T02:54:21.363 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:21.364 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:21.653 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph 2024-04-05T02:54:22.228 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:41 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:22.381 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================== 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================== 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout: ceph x86_64 2:17.2.7-815.g93c790e1.el9 ceph 6.5 k 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mds x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.1 M 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr x86_64 2:17.2.7-815.g93c790e1.el9 ceph 1.4 M 2024-04-05T02:54:22.384 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-modules-core noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 240 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mon x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.9 M 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-osd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 15 M 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-05T02:54:22.385 INFO:teuthology.orchestra.run.smithi151.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-04-05T02:54:22.386 INFO:teuthology.orchestra.run.smithi151.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================== 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout:Install 33 Packages 2024-04-05T02:54:22.387 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:22.388 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 27 M 2024-04-05T02:54:22.388 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 96 M 2024-04-05T02:54:22.388 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:54:22.669 INFO:teuthology.orchestra.run.smithi151.stdout:(1/33): ceph-17.2.7-815.g93c790e1.el9.x86_64.rp 38 kB/s | 6.5 kB 00:00 2024-04-05T02:54:22.861 INFO:teuthology.orchestra.run.smithi151.stdout:(2/33): ceph-mds-17.2.7-815.g93c790e1.el9.x86_6 5.7 MB/s | 2.1 MB 00:00 2024-04-05T02:54:23.370 INFO:teuthology.orchestra.run.smithi151.stdout:(3/33): ceph-mgr-17.2.7-815.g93c790e1.el9.x86_6 1.6 MB/s | 1.4 MB 00:00 2024-04-05T02:54:23.471 INFO:teuthology.orchestra.run.smithi151.stdout:(4/33): ceph-mgr-modules-core-17.2.7-815.g93c79 2.3 MB/s | 240 kB 00:00 2024-04-05T02:54:23.704 INFO:teuthology.orchestra.run.smithi151.stdout:(5/33): ceph-osd-17.2.7-815.g93c790e1.el9.x86_6 18 MB/s | 15 MB 00:00 2024-04-05T02:54:23.730 INFO:teuthology.orchestra.run.smithi151.stdout:(6/33): python3-mako-1.1.4-6.el9.noarch.rpm 557 kB/s | 144 kB 00:00 2024-04-05T02:54:23.755 INFO:teuthology.orchestra.run.smithi151.stdout:(7/33): python3-markupsafe-1.1.1-12.el9.x86_64. 631 kB/s | 32 kB 00:00 2024-04-05T02:54:23.780 INFO:teuthology.orchestra.run.smithi151.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 943 kB/s | 47 kB 00:00 2024-04-05T02:54:23.806 INFO:teuthology.orchestra.run.smithi151.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 765 kB/s | 39 kB 00:00 2024-04-05T02:54:23.864 INFO:teuthology.orchestra.run.smithi151.stdout:(10/33): ceph-mon-17.2.7-815.g93c790e1.el9.x86_ 3.3 MB/s | 3.9 MB 00:01 2024-04-05T02:54:23.890 INFO:teuthology.orchestra.run.smithi151.stdout:(11/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.2 MB/s | 241 kB 00:00 2024-04-05T02:54:23.940 INFO:teuthology.orchestra.run.smithi151.stdout:(12/33): python3-cryptography-36.0.1-4.el9.x86_ 8.6 MB/s | 1.1 MB 00:00 2024-04-05T02:54:23.957 INFO:teuthology.orchestra.run.smithi151.stdout:(13/33): python3-ply-3.11-14.el9.noarch.rpm 1.1 MB/s | 103 kB 00:00 2024-04-05T02:54:23.982 INFO:teuthology.orchestra.run.smithi151.stdout:(14/33): python3-pycparser-2.20-6.el9.noarch.rp 1.3 MB/s | 124 kB 00:00 2024-04-05T02:54:24.008 INFO:teuthology.orchestra.run.smithi151.stdout:(15/33): python3-urllib3-1.26.5-5.el9.noarch.rp 2.7 MB/s | 187 kB 00:00 2024-04-05T02:54:24.066 INFO:teuthology.orchestra.run.smithi151.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 522 kB/s | 43 kB 00:00 2024-04-05T02:54:24.125 INFO:teuthology.orchestra.run.smithi151.stdout:(17/33): python3-requests-2.25.1-8.el9.noarch.r 880 kB/s | 125 kB 00:00 2024-04-05T02:54:24.150 INFO:teuthology.orchestra.run.smithi151.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.2 MB/s | 172 kB 00:00 2024-04-05T02:54:24.176 INFO:teuthology.orchestra.run.smithi151.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 213 kB/s | 11 kB 00:00 2024-04-05T02:54:24.201 INFO:teuthology.orchestra.run.smithi151.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 352 kB/s | 18 kB 00:00 2024-04-05T02:54:24.226 INFO:teuthology.orchestra.run.smithi151.stdout:(21/33): python3-cherrypy-18.6.1-2.el9.noarch.r 2.2 MB/s | 358 kB 00:00 2024-04-05T02:54:24.252 INFO:teuthology.orchestra.run.smithi151.stdout:(22/33): python3-jaraco-collections-3.0.0-8.el9 306 kB/s | 23 kB 00:00 2024-04-05T02:54:24.277 INFO:teuthology.orchestra.run.smithi151.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 257 kB/s | 19 kB 00:00 2024-04-05T02:54:24.302 INFO:teuthology.orchestra.run.smithi151.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 261 kB/s | 20 kB 00:00 2024-04-05T02:54:24.328 INFO:teuthology.orchestra.run.smithi151.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 919 kB/s | 46 kB 00:00 2024-04-05T02:54:24.386 INFO:teuthology.orchestra.run.smithi151.stdout:(26/33): python3-pecan-1.4.2-3.el9.noarch.rpm 3.2 MB/s | 272 kB 00:00 2024-04-05T02:54:24.412 INFO:teuthology.orchestra.run.smithi151.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 196 kB/s | 16 kB 00:00 2024-04-05T02:54:24.437 INFO:teuthology.orchestra.run.smithi151.stdout:(28/33): python3-more-itertools-8.12.0-2.el9.no 494 kB/s | 79 kB 00:00 2024-04-05T02:54:24.462 INFO:teuthology.orchestra.run.smithi151.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 715 kB/s | 36 kB 00:00 2024-04-05T02:54:24.479 INFO:teuthology.orchestra.run.smithi151.stdout:(30/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.3 MB/s | 90 kB 00:00 2024-04-05T02:54:24.513 INFO:teuthology.orchestra.run.smithi151.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-04-05T02:54:24.530 INFO:teuthology.orchestra.run.smithi151.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 399 kB/s | 20 kB 00:00 2024-04-05T02:54:24.613 INFO:teuthology.orchestra.run.smithi151.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.8 MB/s | 427 kB 00:00 2024-04-05T02:54:24.617 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:54:24.617 INFO:teuthology.orchestra.run.smithi151.stdout:Total 12 MB/s | 27 MB 00:02 2024-04-05T02:54:24.632 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:54:24.672 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:54:24.672 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:54:25.014 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:54:25.015 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:54:26.282 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:54:26.333 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-05T02:54:26.365 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-05T02:54:26.399 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-05T02:54:26.491 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-05T02:54:26.561 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-05T02:54:26.602 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-05T02:54:26.674 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-05T02:54:26.728 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-04-05T02:54:26.766 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-04-05T02:54:26.815 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-04-05T02:54:26.872 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-04-05T02:54:27.063 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-04-05T02:54:27.105 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-04-05T02:54:27.144 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-04-05T02:54:27.187 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-04-05T02:54:27.218 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-05T02:54:27.294 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-04-05T02:54:27.328 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:54:27.361 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-04-05T02:54:27.455 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-04-05T02:54:27.504 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-04-05T02:54:27.538 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-04-05T02:54:27.629 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-04-05T02:54:27.669 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-04-05T02:54:27.743 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-04-05T02:54:27.881 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-04-05T02:54:28.007 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:54:28.161 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 28/33 2024-04-05T02:54:28.184 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:54:28.209 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:54:28.209 INFO:teuthology.orchestra.run.smithi151.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:54:28.209 INFO:teuthology.orchestra.run.smithi151.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-05T02:54:28.209 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:54:28.209 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:54:28.209 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:29.587 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:54:29.616 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:54:29.616 INFO:teuthology.orchestra.run.smithi151.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:54:29.616 INFO:teuthology.orchestra.run.smithi151.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-05T02:54:29.616 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:54:29.616 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:54:29.616 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:29.996 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:54:30.021 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:54:30.021 INFO:teuthology.orchestra.run.smithi151.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:54:30.021 INFO:teuthology.orchestra.run.smithi151.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-05T02:54:30.021 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:54:30.021 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:54:30.021 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:30.230 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:54:30.254 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:54:30.255 INFO:teuthology.orchestra.run.smithi151.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:54:30.255 INFO:teuthology.orchestra.run.smithi151.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-05T02:54:30.255 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:54:30.255 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:54:30.255 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:30.319 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 1/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 3/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 4/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 5/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 6/33 2024-04-05T02:54:31.960 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-05T02:54:31.961 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-05T02:54:31.962 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-05T02:54:32.924 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-05T02:54:32.924 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:32.924 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-05T02:54:32.925 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-ply-3.11-14.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-05T02:54:32.926 INFO:teuthology.orchestra.run.smithi151.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-05T02:54:32.927 INFO:teuthology.orchestra.run.smithi151.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-05T02:54:32.927 INFO:teuthology.orchestra.run.smithi151.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-05T02:54:32.927 INFO:teuthology.orchestra.run.smithi151.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-05T02:54:32.927 INFO:teuthology.orchestra.run.smithi151.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-05T02:54:32.927 INFO:teuthology.orchestra.run.smithi151.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-05T02:54:32.927 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:32.927 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:33.273 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-base 2024-04-05T02:54:33.850 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:52 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:33.947 INFO:teuthology.orchestra.run.smithi151.stdout:Package ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:54:33.998 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:33.999 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:54:34.000 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:34.067 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install cephadm 2024-04-05T02:54:34.634 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:53 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:34.786 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:34.787 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:34.787 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:54:34.787 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:34.787 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:54:34.787 INFO:teuthology.orchestra.run.smithi151.stdout: cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 82 k 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 82 k 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 372 k 2024-04-05T02:54:34.788 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:54:35.017 INFO:teuthology.orchestra.run.smithi151.stdout:cephadm-17.2.7-815.g93c790e1.el9.noarch.rpm 359 kB/s | 82 kB 00:00 2024-04-05T02:54:35.017 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:54:35.017 INFO:teuthology.orchestra.run.smithi151.stdout:Total 357 kB/s | 82 kB 00:00 2024-04-05T02:54:35.018 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:54:35.022 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:54:35.023 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:54:35.024 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:54:35.025 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:54:35.104 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:54:35.575 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:54:35.682 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:54:36.084 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:54:36.381 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:54:36.381 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:36.381 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:54:36.382 INFO:teuthology.orchestra.run.smithi151.stdout: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:54:36.382 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:36.382 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:36.583 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-immutable-object-cache 2024-04-05T02:54:37.160 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:56 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:37.317 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:37.318 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:37.318 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2024-04-05T02:54:37.318 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:37.318 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:54:37.318 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-815.g93c790e1.el9 ceph 137 k 2024-04-05T02:54:37.318 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:37.319 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:54:37.319 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:37.319 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2024-04-05T02:54:37.319 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:37.319 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 137 k 2024-04-05T02:54:37.319 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 379 k 2024-04-05T02:54:37.319 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:54:37.655 INFO:teuthology.orchestra.run.smithi151.stdout:ceph-immutable-object-cache-17.2.7-815.g93c790e 409 kB/s | 137 kB 00:00 2024-04-05T02:54:37.655 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:54:37.656 INFO:teuthology.orchestra.run.smithi151.stdout:Total 407 kB/s | 137 kB 00:00 2024-04-05T02:54:37.656 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:54:37.663 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:54:37.663 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:54:37.716 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:54:37.716 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:54:37.855 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:54:37.914 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:54:37.938 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:54:37.938 INFO:teuthology.orchestra.run.smithi151.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:54:37.938 INFO:teuthology.orchestra.run.smithi151.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-05T02:54:37.938 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:38.946 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:54:38.946 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:38.947 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:54:38.947 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:54:38.947 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:38.947 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:39.170 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr 2024-04-05T02:54:39.752 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:58 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:39.855 INFO:teuthology.orchestra.run.smithi151.stdout:Package ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:54:39.907 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:39.908 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:54:39.908 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:39.972 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-dashboard 2024-04-05T02:54:40.535 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:00:59 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:40.685 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-dashboard noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 1.7 M 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-grafana-dashboards noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 24 k 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-prometheus-alerts noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 15 k 2024-04-05T02:54:40.686 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout:Installing weak dependencies: 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout:Install 7 Packages 2024-04-05T02:54:40.687 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:40.688 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 2.0 M 2024-04-05T02:54:40.688 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 8.1 M 2024-04-05T02:54:40.688 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:54:40.929 INFO:teuthology.orchestra.run.smithi151.stdout:(1/7): ceph-prometheus-alerts-17.2.7-815.g93c79 77 kB/s | 15 kB 00:00 2024-04-05T02:54:40.971 INFO:teuthology.orchestra.run.smithi151.stdout:(2/7): ceph-grafana-dashboards-17.2.7-815.g93c7 103 kB/s | 24 kB 00:00 2024-04-05T02:54:41.021 INFO:teuthology.orchestra.run.smithi151.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 98 kB/s | 9.0 kB 00:00 2024-04-05T02:54:41.113 INFO:teuthology.orchestra.run.smithi151.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 335 kB/s | 31 kB 00:00 2024-04-05T02:54:41.139 INFO:teuthology.orchestra.run.smithi151.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 243 kB/s | 41 kB 00:00 2024-04-05T02:54:41.264 INFO:teuthology.orchestra.run.smithi151.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 1.2 MB/s | 188 kB 00:00 2024-04-05T02:54:41.306 INFO:teuthology.orchestra.run.smithi151.stdout:(7/7): ceph-mgr-dashboard-17.2.7-815.g93c790e1. 3.0 MB/s | 1.7 MB 00:00 2024-04-05T02:54:41.308 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:54:41.308 INFO:teuthology.orchestra.run.smithi151.stdout:Total 3.2 MB/s | 2.0 MB 00:00 2024-04-05T02:54:41.308 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:54:41.316 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:54:41.316 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:54:41.380 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:54:41.381 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:54:41.674 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:54:41.749 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-05T02:54:41.775 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-05T02:54:41.813 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-05T02:54:41.845 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-05T02:54:41.874 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 5/7 2024-04-05T02:54:42.103 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 6/7 2024-04-05T02:54:42.136 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:54:42.741 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:54:42.741 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 1/7 2024-04-05T02:54:42.741 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2/7 2024-04-05T02:54:42.741 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 3/7 2024-04-05T02:54:42.741 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-05T02:54:42.741 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-05T02:54:42.741 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-05T02:54:43.042 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:43.043 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:43.278 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-05T02:54:43.844 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:02 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:43.997 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:43.998 INFO:teuthology.orchestra.run.smithi151.stdout:=========================================================================================== 2024-04-05T02:54:43.998 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:54:43.998 INFO:teuthology.orchestra.run.smithi151.stdout:=========================================================================================== 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 7.4 M 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-05T02:54:43.999 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout:=========================================================================================== 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout:Install 12 Packages 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:44.000 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 37 M 2024-04-05T02:54:44.001 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 209 M 2024-04-05T02:54:44.001 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:54:44.715 INFO:teuthology.orchestra.run.smithi151.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 132 kB/s | 32 kB 00:00 2024-04-05T02:54:44.757 INFO:teuthology.orchestra.run.smithi151.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 413 kB/s | 17 kB 00:00 2024-04-05T02:54:44.824 INFO:teuthology.orchestra.run.smithi151.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 514 kB/s | 34 kB 00:00 2024-04-05T02:54:44.958 INFO:teuthology.orchestra.run.smithi151.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 6.2 MB/s | 3.0 MB 00:00 2024-04-05T02:54:45.208 INFO:teuthology.orchestra.run.smithi151.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2024-04-05T02:54:45.317 INFO:teuthology.orchestra.run.smithi151.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 13 MB/s | 4.8 MB 00:00 2024-04-05T02:54:45.342 INFO:teuthology.orchestra.run.smithi151.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 1.6 MB/s | 217 kB 00:00 2024-04-05T02:54:45.809 INFO:teuthology.orchestra.run.smithi151.stdout:(8/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 32 MB/s | 16 MB 00:00 2024-04-05T02:54:45.927 INFO:teuthology.orchestra.run.smithi151.stdout:(9/12): ceph-mgr-diskprediction-local-17.2.7-81 5.1 MB/s | 7.4 MB 00:01 2024-04-05T02:54:46.143 INFO:teuthology.orchestra.run.smithi151.stdout:(10/12): python3-devel-3.9.17-2.el9.x86_64.rpm 1.1 MB/s | 245 kB 00:00 2024-04-05T02:54:46.710 INFO:teuthology.orchestra.run.smithi151.stdout:(11/12): libquadmath-11.4.1-3.el9.x86_64.rpm 217 kB/s | 195 kB 00:00 2024-04-05T02:54:47.169 INFO:teuthology.orchestra.run.smithi151.stdout:(12/12): libgfortran-11.4.1-3.el9.x86_64.rpm 440 kB/s | 803 kB 00:01 2024-04-05T02:54:47.172 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:54:47.172 INFO:teuthology.orchestra.run.smithi151.stdout:Total 12 MB/s | 37 MB 00:03 2024-04-05T02:54:47.216 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:54:47.242 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:54:47.242 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:54:47.682 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:54:47.682 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:54:48.392 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:54:48.458 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-04-05T02:54:48.490 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-05T02:54:48.590 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:54:48.635 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-05T02:54:48.816 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:54:48.843 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:54:48.922 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-05T02:54:49.585 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-05T02:54:49.701 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-05T02:54:50.940 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-05T02:54:51.972 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-05T02:54:52.000 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 1/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:54:54.702 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-04-05T02:54:54.703 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-04-05T02:54:54.703 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-04-05T02:54:54.703 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-04-05T02:54:54.703 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-04-05T02:54:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-04-05T02:54:55.118 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:55.118 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-05T02:54:55.119 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:55.120 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:54:55.510 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-rook 2024-04-05T02:54:56.090 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:15 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:54:56.240 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================= 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================= 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-rook noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 47 k 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-05T02:54:56.242 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================= 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout:Install 13 Packages 2024-04-05T02:54:56.243 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:54:56.244 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 2.0 M 2024-04-05T02:54:56.244 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 26 M 2024-04-05T02:54:56.244 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:54:56.730 INFO:teuthology.orchestra.run.smithi151.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 191 kB/s | 24 kB 00:00 2024-04-05T02:54:56.746 INFO:teuthology.orchestra.run.smithi151.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 125 kB/s | 18 kB 00:00 2024-04-05T02:54:56.797 INFO:teuthology.orchestra.run.smithi151.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.6 MB/s | 174 kB 00:00 2024-04-05T02:54:56.831 INFO:teuthology.orchestra.run.smithi151.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.5 MB/s | 132 kB 00:00 2024-04-05T02:54:56.856 INFO:teuthology.orchestra.run.smithi151.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 3.5 MB/s | 211 kB 00:00 2024-04-05T02:54:56.873 INFO:teuthology.orchestra.run.smithi151.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 43 kB 00:00 2024-04-05T02:54:56.899 INFO:teuthology.orchestra.run.smithi151.stdout:(7/13): ceph-mgr-rook-17.2.7-815.g93c790e1.el9. 160 kB/s | 47 kB 00:00 2024-04-05T02:54:57.041 INFO:teuthology.orchestra.run.smithi151.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 174 kB/s | 32 kB 00:00 2024-04-05T02:54:57.074 INFO:teuthology.orchestra.run.smithi151.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 74 kB/s | 15 kB 00:00 2024-04-05T02:54:57.142 INFO:teuthology.orchestra.run.smithi151.stdout:(10/13): python3-google-auth-2.28.2-1.el9.noarc 1.0 MB/s | 203 kB 00:00 2024-04-05T02:54:57.209 INFO:teuthology.orchestra.run.smithi151.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 441 kB/s | 59 kB 00:00 2024-04-05T02:54:57.242 INFO:teuthology.orchestra.run.smithi151.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 896 kB/s | 90 kB 00:00 2024-04-05T02:54:57.276 INFO:teuthology.orchestra.run.smithi151.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 4.4 MB/s | 1.0 MB 00:00 2024-04-05T02:54:57.281 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:54:57.281 INFO:teuthology.orchestra.run.smithi151.stdout:Total 2.0 MB/s | 2.0 MB 00:01 2024-04-05T02:54:57.281 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:54:57.294 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:54:57.294 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:54:57.389 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:54:57.389 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:54:57.909 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:54:58.091 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-05T02:54:58.153 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-05T02:54:58.217 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-05T02:54:58.254 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-05T02:54:58.295 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-05T02:54:58.390 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-05T02:54:58.508 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-05T02:54:58.564 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-05T02:54:59.307 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-05T02:54:59.363 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-05T02:54:59.397 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-05T02:54:59.443 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-05T02:54:59.467 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:55:00.323 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 1/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-05T02:55:00.324 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-05T02:55:00.675 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-05T02:55:00.676 INFO:teuthology.orchestra.run.smithi151.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-05T02:55:00.677 INFO:teuthology.orchestra.run.smithi151.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-05T02:55:00.677 INFO:teuthology.orchestra.run.smithi151.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-05T02:55:00.677 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:00.677 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:00.907 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-mgr-cephadm 2024-04-05T02:55:01.479 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:20 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:01.631 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================= 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================= 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 123 k 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-05T02:55:01.632 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:01.633 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:55:01.633 INFO:teuthology.orchestra.run.smithi151.stdout:======================================================================================= 2024-04-05T02:55:01.633 INFO:teuthology.orchestra.run.smithi151.stdout:Install 6 Packages 2024-04-05T02:55:01.633 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:01.633 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 6.8 M 2024-04-05T02:55:01.634 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 33 M 2024-04-05T02:55:01.634 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:55:01.883 INFO:teuthology.orchestra.run.smithi151.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.6 MB/s | 248 kB 00:00 2024-04-05T02:55:01.959 INFO:teuthology.orchestra.run.smithi151.stdout:(2/6): ceph-mgr-cephadm-17.2.7-815.g93c790e1.el 536 kB/s | 123 kB 00:00 2024-04-05T02:55:02.109 INFO:teuthology.orchestra.run.smithi151.stdout:(3/6): python3-babel-2.9.1-2.el9.noarch.rpm 15 MB/s | 5.8 MB 00:00 2024-04-05T02:55:02.368 INFO:teuthology.orchestra.run.smithi151.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 151 kB/s | 58 kB 00:00 2024-04-05T02:55:02.402 INFO:teuthology.orchestra.run.smithi151.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 190 kB/s | 51 kB 00:00 2024-04-05T02:55:02.427 INFO:teuthology.orchestra.run.smithi151.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.0 MB/s | 548 kB 00:00 2024-04-05T02:55:02.430 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:02.430 INFO:teuthology.orchestra.run.smithi151.stdout:Total 8.5 MB/s | 6.8 MB 00:00 2024-04-05T02:55:02.444 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:55:02.454 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:55:02.454 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:55:02.543 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:55:02.545 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:55:02.841 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:55:03.013 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-05T02:55:03.063 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-05T02:55:03.589 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-05T02:55:03.674 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-05T02:55:03.739 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-05T02:55:03.768 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:55:04.571 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:55:04.572 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/6 2024-04-05T02:55:04.572 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-05T02:55:04.572 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-05T02:55:04.572 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-05T02:55:04.572 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-05T02:55:04.881 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-05T02:55:04.881 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:04.881 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:04.881 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:55:04.881 INFO:teuthology.orchestra.run.smithi151.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-05T02:55:04.882 INFO:teuthology.orchestra.run.smithi151.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-05T02:55:04.882 INFO:teuthology.orchestra.run.smithi151.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-05T02:55:04.882 INFO:teuthology.orchestra.run.smithi151.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-05T02:55:04.882 INFO:teuthology.orchestra.run.smithi151.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-05T02:55:04.882 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:04.882 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:05.143 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install ceph-fuse 2024-04-05T02:55:05.712 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:24 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:05.865 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:05.865 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repository Size 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 812 k 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout:Install 2 Packages 2024-04-05T02:55:05.866 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:05.867 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 891 k 2024-04-05T02:55:05.867 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 2.6 M 2024-04-05T02:55:05.867 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:55:06.007 INFO:teuthology.orchestra.run.smithi151.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 563 kB/s | 79 kB 00:00 2024-04-05T02:55:06.324 INFO:teuthology.orchestra.run.smithi151.stdout:(2/2): ceph-fuse-17.2.7-815.g93c790e1.el9.x86_6 1.7 MB/s | 812 kB 00:00 2024-04-05T02:55:06.325 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:06.326 INFO:teuthology.orchestra.run.smithi151.stdout:Total 1.9 MB/s | 891 kB 00:00 2024-04-05T02:55:06.326 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:55:06.347 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:55:06.347 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:55:06.420 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:55:06.420 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:55:06.632 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:55:06.762 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-05T02:55:06.825 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:55:07.550 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:55:07.550 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/2 2024-04-05T02:55:07.874 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-05T02:55:07.874 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:07.874 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:07.874 INFO:teuthology.orchestra.run.smithi151.stdout: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-05T02:55:07.874 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:07.874 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:08.159 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install librados-devel 2024-04-05T02:55:08.730 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:27 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:08.886 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout: librados-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 126 k 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 126 k 2024-04-05T02:55:08.887 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 448 k 2024-04-05T02:55:08.888 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:55:09.233 INFO:teuthology.orchestra.run.smithi151.stdout:librados-devel-17.2.7-815.g93c790e1.el9.x86_64. 365 kB/s | 126 kB 00:00 2024-04-05T02:55:09.233 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:09.234 INFO:teuthology.orchestra.run.smithi151.stdout:Total 364 kB/s | 126 kB 00:00 2024-04-05T02:55:09.234 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:55:09.239 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:55:09.239 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:55:09.294 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:55:09.294 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:55:09.454 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:55:09.556 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:10.008 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:10.281 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:10.281 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:10.281 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:10.281 INFO:teuthology.orchestra.run.smithi151.stdout: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:55:10.281 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:10.281 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:10.540 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install libcephfs2 2024-04-05T02:55:11.137 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:30 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:11.240 INFO:teuthology.orchestra.run.smithi151.stdout:Package libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:55:11.294 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:11.295 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:55:11.296 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:11.365 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install libcephfs-devel 2024-04-05T02:55:11.933 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:30 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:12.084 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:12.085 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:12.085 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2024-04-05T02:55:12.085 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:12.085 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:55:12.085 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:55:12.085 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:12.086 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:55:12.086 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:12.086 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2024-04-05T02:55:12.086 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:12.086 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 25 k 2024-04-05T02:55:12.086 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 110 k 2024-04-05T02:55:12.086 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:55:12.278 INFO:teuthology.orchestra.run.smithi151.stdout:libcephfs-devel-17.2.7-815.g93c790e1.el9.x86_64 130 kB/s | 25 kB 00:00 2024-04-05T02:55:12.278 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:12.278 INFO:teuthology.orchestra.run.smithi151.stdout:Total 129 kB/s | 25 kB 00:00 2024-04-05T02:55:12.279 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:55:12.282 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:55:12.282 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:55:12.283 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:55:12.284 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:55:12.371 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:55:12.472 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:12.874 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:13.137 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:13.138 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:13.138 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:13.138 INFO:teuthology.orchestra.run.smithi151.stdout: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:55:13.138 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:13.138 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:13.297 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install librados2 2024-04-05T02:55:13.865 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:32 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:13.964 INFO:teuthology.orchestra.run.smithi151.stdout:Package librados2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:55:14.018 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:14.018 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:55:14.019 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:14.087 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install librbd1 2024-04-05T02:55:14.650 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:33 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:14.750 INFO:teuthology.orchestra.run.smithi151.stdout:Package librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:55:14.803 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:14.804 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:55:14.804 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:14.879 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-rados 2024-04-05T02:55:15.446 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:34 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:15.553 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:55:15.605 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:15.606 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:55:15.606 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:15.681 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-rgw 2024-04-05T02:55:16.252 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:35 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:16.350 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:55:16.402 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:16.403 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:55:16.403 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:16.479 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-cephfs 2024-04-05T02:55:17.043 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:36 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:17.142 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:55:17.194 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:17.195 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:55:17.195 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:17.271 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install python3-rbd 2024-04-05T02:55:17.835 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:36 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:17.933 INFO:teuthology.orchestra.run.smithi151.stdout:Package python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:55:17.985 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:17.986 INFO:teuthology.orchestra.run.smithi151.stdout:Nothing to do. 2024-04-05T02:55:17.987 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:18.059 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install rbd-fuse 2024-04-05T02:55:18.620 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:37 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:18.769 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:18.770 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:18.770 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2024-04-05T02:55:18.770 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:18.770 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:55:18.770 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 82 k 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 82 k 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 222 k 2024-04-05T02:55:18.771 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:55:19.061 INFO:teuthology.orchestra.run.smithi151.stdout:rbd-fuse-17.2.7-815.g93c790e1.el9.x86_64.rpm 282 kB/s | 82 kB 00:00 2024-04-05T02:55:19.062 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:19.062 INFO:teuthology.orchestra.run.smithi151.stdout:Total 281 kB/s | 82 kB 00:00 2024-04-05T02:55:19.062 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:55:19.069 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:55:19.069 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:55:19.123 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:55:19.124 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:55:19.271 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:55:19.376 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:19.843 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:20.141 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:20.141 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:20.141 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:20.141 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:55:20.141 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:20.141 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:20.380 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install rbd-mirror 2024-04-05T02:55:20.959 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:39 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:21.115 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout: Package Arch Version Repo Size 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-mirror x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2024-04-05T02:55:21.116 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:21.117 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 2.9 M 2024-04-05T02:55:21.117 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 12 M 2024-04-05T02:55:21.117 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:55:21.501 INFO:teuthology.orchestra.run.smithi151.stdout:rbd-mirror-17.2.7-815.g93c790e1.el9.x86_64.rpm 7.6 MB/s | 2.9 MB 00:00 2024-04-05T02:55:21.502 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:21.502 INFO:teuthology.orchestra.run.smithi151.stdout:Total 7.6 MB/s | 2.9 MB 00:00 2024-04-05T02:55:21.502 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:55:21.512 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:55:21.512 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:55:21.585 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:55:21.585 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:55:21.988 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:55:22.044 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:22.069 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:22.069 INFO:teuthology.orchestra.run.smithi151.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:55:22.069 INFO:teuthology.orchestra.run.smithi151.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-05T02:55:22.069 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:55:22.070 INFO:teuthology.orchestra.run.smithi151.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:55:22.070 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:23.269 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:23.269 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:55:23.270 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:23.270 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:23.484 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum -y install rbd-nbd 2024-04-05T02:55:24.066 INFO:teuthology.orchestra.run.smithi151.stdout:Last metadata expiration check: 0:01:43 ago on Fri 05 Apr 2024 02:53:41 AM UTC. 2024-04-05T02:55:24.218 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-05T02:55:24.218 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-nbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 165 k 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:Install 1 Package 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 165 k 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 477 k 2024-04-05T02:55:24.219 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-05T02:55:24.450 INFO:teuthology.orchestra.run.smithi151.stdout:rbd-nbd-17.2.7-815.g93c790e1.el9.x86_64.rpm 715 kB/s | 165 kB 00:00 2024-04-05T02:55:24.451 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:24.451 INFO:teuthology.orchestra.run.smithi151.stdout:Total 710 kB/s | 165 kB 00:00 2024-04-05T02:55:24.451 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-05T02:55:24.459 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-05T02:55:24.459 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-05T02:55:24.515 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-05T02:55:24.515 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-05T02:55:24.670 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-05T02:55:24.782 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:25.219 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:25.535 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:55:25.535 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:25.535 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-05T02:55:25.536 INFO:teuthology.orchestra.run.smithi151.stdout: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:55:25.536 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:55:25.536 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-05T02:55:25.769 DEBUG:teuthology.parallel:result is None 2024-04-05T02:55:25.769 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:55:25.770 INFO:teuthology.packaging:ref: None 2024-04-05T02:55:25.770 INFO:teuthology.packaging:tag: None 2024-04-05T02:55:25.770 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:55:25.770 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:55:25.770 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:55:26.080 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-05T02:55:26.106 INFO:teuthology.orchestra.run.smithi033.stdout:17.2.7-815.g93c790e1.el9 2024-04-05T02:55:26.106 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-815.g93c790e1.el9 2024-04-05T02:55:26.106 INFO:teuthology.task.install:The correct ceph version 17.2.7-815.g93c790e1 is installed. 2024-04-05T02:55:26.108 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:55:26.108 INFO:teuthology.packaging:ref: None 2024-04-05T02:55:26.108 INFO:teuthology.packaging:tag: None 2024-04-05T02:55:26.108 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:55:26.108 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:55:26.108 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:55:26.393 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-05T02:55:26.414 INFO:teuthology.orchestra.run.smithi151.stdout:17.2.7-815.g93c790e1.el9 2024-04-05T02:55:26.414 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-815.g93c790e1.el9 2024-04-05T02:55:26.414 INFO:teuthology.task.install:The correct ceph version 17.2.7-815.g93c790e1 is installed. 2024-04-05T02:55:26.416 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-05T02:55:26.416 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:55:26.416 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-05T02:55:26.449 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:55:26.449 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-05T02:55:26.483 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-05T02:55:26.483 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:55:26.484 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/daemon-helper 2024-04-05T02:55:26.519 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-05T02:55:26.589 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:55:26.590 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/usr/bin/daemon-helper 2024-04-05T02:55:26.617 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-05T02:55:26.684 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-05T02:55:26.684 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:55:26.685 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-05T02:55:26.714 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-05T02:55:26.784 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:55:26.784 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-05T02:55:26.811 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-05T02:55:26.878 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-05T02:55:26.878 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:55:26.879 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/stdin-killer 2024-04-05T02:55:26.908 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-05T02:55:26.978 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:55:26.979 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/usr/bin/stdin-killer 2024-04-05T02:55:27.006 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-05T02:55:27.073 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-05T02:55:27.181 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06'} 2024-04-05T02:55:27.182 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-05T02:55:27.182 INFO:tasks.cephadm:Cluster fsid is f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:55:27.182 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-05T02:55:27.182 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.33', 'mon.c': '[v2:172.21.15.33:3301,v1:172.21.15.33:6790]', 'mon.b': '172.21.15.151'} 2024-04-05T02:55:27.183 INFO:tasks.cephadm:First mon is mon.a on smithi033 2024-04-05T02:55:27.183 INFO:tasks.cephadm:First mgr is y 2024-04-05T02:55:27.183 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-05T02:55:27.183 DEBUG:teuthology.orchestra.run.smithi033:> sudo hostname $(hostname -s) 2024-04-05T02:55:27.215 DEBUG:teuthology.orchestra.run.smithi151:> sudo hostname $(hostname -s) 2024-04-05T02:55:27.242 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-05T02:55:27.243 DEBUG:teuthology.orchestra.run.smithi033:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-05T02:55:27.472 INFO:teuthology.orchestra.run.smithi033.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 5 02:55 /home/ubuntu/cephtest/cephadm 2024-04-05T02:55:27.472 DEBUG:teuthology.orchestra.run.smithi151:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-05T02:55:27.685 INFO:teuthology.orchestra.run.smithi151.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 5 02:55 /home/ubuntu/cephtest/cephadm 2024-04-05T02:55:27.685 DEBUG:teuthology.orchestra.run.smithi033:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-05T02:55:27.704 DEBUG:teuthology.orchestra.run.smithi151:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-05T02:55:27.731 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-05T02:55:27.731 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-05T02:55:27.747 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-05T02:55:28.560 INFO:teuthology.orchestra.run.smithi033.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:55:28.691 INFO:teuthology.orchestra.run.smithi151.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:55:54.808 INFO:teuthology.orchestra.run.smithi151.stdout:{ 2024-04-05T02:55:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "ceph_version": "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)", 2024-04-05T02:55:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "image_id": "a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48", 2024-04-05T02:55:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "repo_digests": [ 2024-04-05T02:55:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030" 2024-04-05T02:55:54.809 INFO:teuthology.orchestra.run.smithi151.stdout: ] 2024-04-05T02:55:54.809 INFO:teuthology.orchestra.run.smithi151.stdout:} 2024-04-05T02:55:55.655 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-04-05T02:55:55.655 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_version": "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)", 2024-04-05T02:55:55.655 INFO:teuthology.orchestra.run.smithi033.stdout: "image_id": "a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48", 2024-04-05T02:55:55.655 INFO:teuthology.orchestra.run.smithi033.stdout: "repo_digests": [ 2024-04-05T02:55:55.655 INFO:teuthology.orchestra.run.smithi033.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030" 2024-04-05T02:55:55.655 INFO:teuthology.orchestra.run.smithi033.stdout: ] 2024-04-05T02:55:55.655 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-04-05T02:55:55.678 DEBUG:teuthology.orchestra.run.smithi033:> sudo mkdir -p /etc/ceph 2024-04-05T02:55:55.711 DEBUG:teuthology.orchestra.run.smithi151:> sudo mkdir -p /etc/ceph 2024-04-05T02:55:55.746 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 777 /etc/ceph 2024-04-05T02:55:55.777 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 777 /etc/ceph 2024-04-05T02:55:55.817 INFO:tasks.cephadm:Writing seed config... 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-05T02:55:55.818 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-05T02:55:55.819 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:55:55.819 DEBUG:teuthology.orchestra.run.smithi033:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-05T02:55:55.834 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = f4bb5524-f2f7-11ee-b649-cb9ed24678a4 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-05T02:55:55.834 DEBUG:teuthology.orchestra.run.smithi033:mon.a> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service 2024-04-05T02:55:55.879 DEBUG:teuthology.orchestra.run.smithi033:mgr.y> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service 2024-04-05T02:55:55.922 INFO:tasks.cephadm:Bootstrapping... 2024-04-05T02:55:55.923 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.33 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-05T02:55:56.134 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:55:56.134 INFO:teuthology.orchestra.run.smithi033.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'f4bb5524-f2f7-11ee-b649-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.33', '--skip-admin-label'] 2024-04-05T02:55:56.160 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-05T02:55:56.160 INFO:teuthology.orchestra.run.smithi033.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-05T02:55:56.161 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying podman|docker is present... 2024-04-05T02:55:56.184 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-05T02:55:56.184 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying lvm2 is present... 2024-04-05T02:55:56.184 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying time synchronization is in place... 2024-04-05T02:55:56.192 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-05T02:55:56.192 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-05T02:55:56.198 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-05T02:55:56.198 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-04-05T02:55:56.204 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout enabled 2024-04-05T02:55:56.211 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout active 2024-04-05T02:55:56.211 INFO:teuthology.orchestra.run.smithi033.stdout:Unit chronyd.service is enabled and running 2024-04-05T02:55:56.211 INFO:teuthology.orchestra.run.smithi033.stdout:Repeating the final host check... 2024-04-05T02:55:56.234 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-05T02:55:56.234 INFO:teuthology.orchestra.run.smithi033.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-05T02:55:56.234 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl is present 2024-04-05T02:55:56.234 INFO:teuthology.orchestra.run.smithi033.stdout:lvcreate is present 2024-04-05T02:55:56.240 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-05T02:55:56.241 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-05T02:55:56.247 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-05T02:55:56.247 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-04-05T02:55:56.253 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout enabled 2024-04-05T02:55:56.259 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout active 2024-04-05T02:55:56.260 INFO:teuthology.orchestra.run.smithi033.stdout:Unit chronyd.service is enabled and running 2024-04-05T02:55:56.260 INFO:teuthology.orchestra.run.smithi033.stdout:Host looks OK 2024-04-05T02:55:56.260 INFO:teuthology.orchestra.run.smithi033.stdout:Cluster fsid: f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:55:56.260 INFO:teuthology.orchestra.run.smithi033.stdout:Acquiring lock 140673910449248 on /run/cephadm/f4bb5524-f2f7-11ee-b649-cb9ed24678a4.lock 2024-04-05T02:55:56.260 INFO:teuthology.orchestra.run.smithi033.stdout:Lock 140673910449248 acquired on /run/cephadm/f4bb5524-f2f7-11ee-b649-cb9ed24678a4.lock 2024-04-05T02:55:56.260 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying IP 172.21.15.33 port 3300 ... 2024-04-05T02:55:56.261 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying IP 172.21.15.33 port 6789 ... 2024-04-05T02:55:56.261 INFO:teuthology.orchestra.run.smithi033.stdout:Base mon IP(s) is [172.21.15.33:3300, 172.21.15.33:6789], mon addrv is [v2:172.21.15.33:3300,v1:172.21.15.33:6789] 2024-04-05T02:55:56.264 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.33 metric 100 2024-04-05T02:55:56.264 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.33 metric 100 2024-04-05T02:55:56.266 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-05T02:55:56.267 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-05T02:55:56.267 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-05T02:55:56.269 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-05T02:55:56.269 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-05T02:55:56.269 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-05T02:55:56.269 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-05T02:55:56.270 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fea5:d294/64 scope link noprefixroute 2024-04-05T02:55:56.270 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-05T02:55:56.271 INFO:teuthology.orchestra.run.smithi033.stdout:Mon IP `172.21.15.33` is in CIDR network `172.21.0.0/20` 2024-04-05T02:55:56.271 INFO:teuthology.orchestra.run.smithi033.stdout:Mon IP `172.21.15.33` is in CIDR network `172.21.0.0/20` 2024-04-05T02:55:56.272 INFO:teuthology.orchestra.run.smithi033.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-05T02:55:56.272 INFO:teuthology.orchestra.run.smithi033.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-05T02:55:56.272 INFO:teuthology.orchestra.run.smithi033.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:55:56.747 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 2024-04-05T02:55:56.748 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:55:56.748 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Getting image source signatures 2024-04-05T02:55:56.748 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Copying blob sha256:7c8e66990dd5fdc46cc6d5e1ff7927cb8fbc9e77ba63bb5152631196a3d52311 2024-04-05T02:55:56.748 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Copying blob sha256:77070339c81bcb5c53bbbf3f31a47f7187e5393cb0e096452852a31192d97570 2024-04-05T02:55:56.748 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Copying config sha256:a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 2024-04-05T02:55:56.748 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-05T02:55:57.580 INFO:teuthology.orchestra.run.smithi033.stdout:ceph: stdout ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable) 2024-04-05T02:55:57.580 INFO:teuthology.orchestra.run.smithi033.stdout:Ceph version: ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable) 2024-04-05T02:55:57.580 INFO:teuthology.orchestra.run.smithi033.stdout:Extracting ceph user uid/gid from container image... 2024-04-05T02:55:58.507 INFO:teuthology.orchestra.run.smithi033.stdout:stat: stdout 167 167 2024-04-05T02:55:58.507 INFO:teuthology.orchestra.run.smithi033.stdout:Creating initial keys... 2024-04-05T02:55:59.663 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph-authtool: stdout AQA+aA9mxmSZNBAAk4Hx8FksHZOiUnDSE3c18w== 2024-04-05T02:56:00.603 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph-authtool: stdout AQA/aA9m+jNZOxAACCXtXBeIyaJJz0nV3UI0Aw== 2024-04-05T02:56:01.374 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph-authtool: stdout AQBAaA9mkVm5NhAAPpU7Z3D+CCLa11gky1YaPw== 2024-04-05T02:56:01.375 INFO:teuthology.orchestra.run.smithi033.stdout:Creating initial monmap... 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:monmaptool for a [v2:172.21.15.33:3300,v1:172.21.15.33:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:setting min_mon_release = octopus 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: set fsid to f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:56:02.214 INFO:teuthology.orchestra.run.smithi033.stdout:Creating mon... 2024-04-05T02:56:03.178 INFO:teuthology.orchestra.run.smithi033.stdout:create mon.a on 2024-04-05T02:56:03.386 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-05T02:56:03.524 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-05T02:56:03.672 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4.target. 2024-04-05T02:56:03.672 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4.target. 2024-04-05T02:56:03.838 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a 2024-04-05T02:56:03.838 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to reset failed state of unit ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service: Unit ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service not loaded. 2024-04-05T02:56:03.997 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4.target.wants/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@.service. 2024-04-05T02:56:04.504 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:04 smithi033 ceph-mon[28870]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-05T02:56:04.505 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:04 smithi033 bash[28851]: 61ad125578956c8f5a9e79aeeb9264e4bbe16234823386d29e49ca7027501996 2024-04-05T02:56:04.518 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:56:04.518 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-04-05T02:56:04.525 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:56:04.526 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-04-05T02:56:04.526 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-04-05T02:56:04.526 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to enable service . firewalld.service is not available 2024-04-05T02:56:04.526 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mon to start... 2024-04-05T02:56:04.526 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mon... 2024-04-05T02:56:04.762 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:04 smithi033 systemd[1]: Started Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T02:56:05.509 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout cluster: 2024-04-05T02:56:05.509 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout id: f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:05.509 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-05T02:56:05.509 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:05.509 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout services: 2024-04-05T02:56:05.509 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.5746s) 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout data: 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout pgs: 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:mon is available 2024-04-05T02:56:05.510 INFO:teuthology.orchestra.run.smithi033.stdout:Assimilating anything we can from ceph.conf... 2024-04-05T02:56:05.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:05 smithi033 ceph-mon[28870]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-05T02:56:05.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:05 smithi033 ceph-mon[28870]: monmap e1: 1 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:56:05.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:05 smithi033 ceph-mon[28870]: fsmap 2024-04-05T02:56:05.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:05 smithi033 ceph-mon[28870]: osdmap e1: 0 total, 0 up, 0 in 2024-04-05T02:56:05.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:05 smithi033 ceph-mon[28870]: mgrmap e1: no daemons active 2024-04-05T02:56:05.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:05 smithi033 ceph-mon[28870]: from='client.? 172.21.15.33:0/266692317' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-05T02:56:06.548 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [global] 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout fsid = f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.33:3300,v1:172.21.15.33:6789] 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [osd] 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-05T02:56:06.549 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-05T02:56:06.550 INFO:teuthology.orchestra.run.smithi033.stdout:Generating new minimal ceph.conf... 2024-04-05T02:56:06.645 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:06 smithi033 ceph-mon[28870]: from='client.? 172.21.15.33:0/3969019502' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-05T02:56:06.645 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:06 smithi033 ceph-mon[28870]: from='client.? 172.21.15.33:0/3969019502' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-05T02:56:07.479 INFO:teuthology.orchestra.run.smithi033.stdout:Restarting the monitor... 2024-04-05T02:56:07.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:07 smithi033 ceph-mon[28870]: from='client.? 172.21.15.33:0/598935998' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:56:07.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:07 smithi033 systemd[1]: Stopping Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T02:56:07.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:07 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a[28866]: 2024-04-05T02:56:07.728+0000 7f805fd31700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T02:56:07.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:07 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a[28866]: 2024-04-05T02:56:07.728+0000 7f805fd31700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-05T02:56:07.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:07 smithi033 podman[29116]: 2024-04-05 02:56:07.739664035 +0000 UTC m=+0.094360431 container died 61ad125578956c8f5a9e79aeeb9264e4bbe16234823386d29e49ca7027501996 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T02:56:08.356 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 podman[29116]: 2024-04-05 02:56:08.038042918 +0000 UTC m=+0.392739320 container cleanup 61ad125578956c8f5a9e79aeeb9264e4bbe16234823386d29e49ca7027501996 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, RELEASE=quincy-6fda41f, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-05T02:56:08.357 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 bash[29116]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a 2024-04-05T02:56:08.357 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 podman[29130]: 2024-04-05 02:56:08.255549962 +0000 UTC m=+0.516186805 container remove 61ad125578956c8f5a9e79aeeb9264e4bbe16234823386d29e49ca7027501996 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-6fda41f, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-04-05T02:56:08.686 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-05T02:56:08.686 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 systemd[1]: Stopped Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T02:56:08.687 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 systemd[1]: Starting Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T02:56:08.687 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 podman[29225]: 2024-04-05 02:56:08.573574686 +0000 UTC m=+0.099786369 container create e876c4ff5784adc7a7db0a8e79e7bb9aeedd1f4a51fdc3cce2573a612daeaf28 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-05T02:56:08.687 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 podman[29225]: 2024-04-05 02:56:08.491634599 +0000 UTC m=+0.017846256 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-05T02:56:08.821 INFO:teuthology.orchestra.run.smithi033.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-05T02:56:08.943 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 podman[29225]: 2024-04-05 02:56:08.750838655 +0000 UTC m=+0.277050295 container init e876c4ff5784adc7a7db0a8e79e7bb9aeedd1f4a51fdc3cce2573a612daeaf28 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T02:56:08.943 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 podman[29225]: 2024-04-05 02:56:08.75369486 +0000 UTC m=+0.279906498 container start e876c4ff5784adc7a7db0a8e79e7bb9aeedd1f4a51fdc3cce2573a612daeaf28 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable), process ceph-mon, pid 2 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: pidfile_write: ignore empty --pid-file 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: load: jerasure load: lrc 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: RocksDB version: 6.15.5 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Compile date Apr 3 2024 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: DB SUMMARY 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: DB Session ID: YLSFDWPIZ1K6C65D5251 2024-04-05T02:56:08.944 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: CURRENT file: CURRENT 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84211 ; 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.error_if_exists: 0 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.create_if_missing: 0 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.paranoid_checks: 1 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.env: 0x556557e66a80 2024-04-05T02:56:08.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.fs: Posix File System 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.info_log: 0x556559324e80 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.statistics: (nil) 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.use_fsync: 0 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_log_file_size: 0 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.allow_fallocate: 1 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T02:56:08.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.use_direct_reads: 0 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.db_log_dir: 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T02:56:08.947 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.write_buffer_manager: 0x55655a094960 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T02:56:08.948 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.unordered_write: 0 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.row_cache: None 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.wal_filter: None 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T02:56:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.preserve_deletes: 0 2024-04-05T02:56:08.950 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.two_write_queues: 0 2024-04-05T02:56:08.950 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T02:56:08.950 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.atomic_flush: 0 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.log_readahead_size: 0 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_background_jobs: 2 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_background_compactions: -1 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_subcompactions: 1 2024-04-05T02:56:08.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_open_files: -1 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_background_flushes: -1 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Compression algorithms supported: 2024-04-05T02:56:08.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kZSTD supported: 0 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kXpressCompression supported: 0 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kZlibCompression supported: 1 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kSnappyCompression supported: 1 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kLZ4Compression supported: 1 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: kBZip2Compression supported: 0 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.merge_operator: 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_filter: None 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T02:56:08.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556559374f58) 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_type: 0 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_index_type: 0 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_shortening: 1 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: hash_index_allow_collision: 1 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: checksum: 1 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: no_block_cache: 0 2024-04-05T02:56:08.954 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache: 0x5565593a7090 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_options: 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: capacity : 536870912 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: num_shard_bits : 4 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: strict_capacity_limit : 0 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_compressed: (nil) 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: persistent_cache: (nil) 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size: 4096 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size_deviation: 10 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_restart_interval: 16 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_block_restart_interval: 1 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: metadata_block_size: 4096 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: partition_filters: 0 2024-04-05T02:56:08.955 INFO:journalctl@ceph.mon.a.smithi033.stdout: use_delta_encoding: 1 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout: whole_key_filtering: 1 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout: verify_compression: 0 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout: format_version: 4 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_index_compression: 1 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_align: 0 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression: NoCompression 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T02:56:08.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.num_levels: 7 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T02:56:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T02:56:08.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.arena_block_size: 4194304 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T02:56:08.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.table_properties_collectors: 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.inplace_update_support: 0 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.bloom_locality: 0 2024-04-05T02:56:08.960 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.max_successive_merges: 0 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.ttl: 2592000 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.enable_blob_files: false 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.min_blob_size: 0 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-05T02:56:08.961 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285768779276, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285768779973, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80643, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78904, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73113, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712285768, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "d841786c-f4a7-4c2d-a11f-c54511b96e84", "db_session_id": "YLSFDWPIZ1K6C65D5251"}} 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285768780309, "job": 1, "event": "recovery_finished"} 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55655933b180 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: DB pointer 0x556559410000 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-05T02:56:08.962 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** DB Stats ** 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: L0 2/0 80.51 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.3 0.00 0.00 1 0.000 0 0 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Sum 2/0 80.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.3 0.00 0.00 1 0.000 0 0 2024-04-05T02:56:08.963 INFO:journalctl@ceph.mon.a.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.3 0.00 0.00 1 0.000 0 0 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.3 0.00 0.00 1 0.000 0 0 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative compaction: 0.00 GB write, 26.73 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval compaction: 0.00 GB write, 26.73 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:56:08.964 INFO:journalctl@ceph.mon.a.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: L0 2/0 80.51 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.3 0.00 0.00 1 0.000 0 0 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: Sum 2/0 80.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.3 0.00 0.00 1 0.000 0 0 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:56:08.965 INFO:journalctl@ceph.mon.a.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.3 0.00 0.00 1 0.000 0 0 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative compaction: 0.00 GB write, 26.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: starting mon.a rank 0 at public addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] at bind addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???) e1 preinit fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???).mds e1 new map 2024-04-05T02:56:08.966 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???).mds e1 print_map 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout: e1 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout: legacy client fscid: -1 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout: No filesystems configured 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 bash[29225]: e876c4ff5784adc7a7db0a8e79e7bb9aeedd1f4a51fdc3cce2573a612daeaf28 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 systemd[1]: Started Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-05T02:56:08.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: monmap e1: 1 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:56:08.968 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: fsmap 2024-04-05T02:56:08.968 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: osdmap e1: 0 total, 0 up, 0 in 2024-04-05T02:56:08.968 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:08 smithi033 ceph-mon[29244]: mgrmap e1: no daemons active 2024-04-05T02:56:09.927 INFO:teuthology.orchestra.run.smithi033.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-05T02:56:09.927 INFO:teuthology.orchestra.run.smithi033.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-05T02:56:09.927 INFO:teuthology.orchestra.run.smithi033.stdout:Creating mgr... 2024-04-05T02:56:09.928 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying port 9283 ... 2024-04-05T02:56:10.105 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y 2024-04-05T02:56:10.105 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to reset failed state of unit ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service: Unit ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service not loaded. 2024-04-05T02:56:10.249 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4.target.wants/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service → /etc/systemd/system/ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@.service. 2024-04-05T02:56:10.581 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:10 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1468938829' entity='client.admin' 2024-04-05T02:56:10.752 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:56:10.752 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-04-05T02:56:10.760 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:56:10.760 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-04-05T02:56:10.760 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-04-05T02:56:10.760 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to enable service . firewalld.service is not available 2024-04-05T02:56:10.768 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:56:10.769 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-04-05T02:56:10.776 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:56:10.776 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-04-05T02:56:10.776 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-04-05T02:56:10.776 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-05T02:56:10.776 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr to start... 2024-04-05T02:56:10.777 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr... 2024-04-05T02:56:11.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:11.199+0000 7fd4088e91c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:56:11.575 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:11 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1737353655' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:56:11.576 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:11.299+0000 7fd4088e91c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsid": "f4bb5524-f2f7-11ee-b649-cb9ed24678a4", 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:11.855 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:11.856 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:56:11.857 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:56:11.858 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:56:04.452703+0000", 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:56:11.859 INFO:teuthology.orchestra.run.smithi033.stdout:mgr not available, waiting (1/15)... 2024-04-05T02:56:12.130 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:11.953+0000 7fd4088e91c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:56:12.131 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.042+0000 7fd4088e91c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:56:12.131 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.127+0000 7fd4088e91c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:56:12.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.277+0000 7fd4088e91c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:56:12.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.358+0000 7fd4088e91c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:56:12.795 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.530+0000 7fd4088e91c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:56:12.796 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.711+0000 7fd4088e91c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:56:12.796 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.795+0000 7fd4088e91c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:56:13.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.889+0000 7fd4088e91c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:56:13.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:12.975+0000 7fd4088e91c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:56:14.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:14.057+0000 7fd4088e91c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:56:14.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:14.190+0000 7fd4088e91c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:56:14.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:14 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2486013203' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:56:14.633 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:14.402+0000 7fd4088e91c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:56:14.633 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:14.632+0000 7fd4088e91c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:56:14.971 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:14.808+0000 7fd4088e91c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:56:15.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:15.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsid": "f4bb5524-f2f7-11ee-b649-cb9ed24678a4", 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:56:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:56:15.007 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:56:15.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:56:15.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:15.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:56:15.013 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:15.013 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:56:04.452703+0000", 2024-04-05T02:56:15.013 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:15.013 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:15.013 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:56:15.013 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:56:15.013 INFO:teuthology.orchestra.run.smithi033.stdout:mgr not available, waiting (2/15)... 2024-04-05T02:56:15.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:14.991+0000 7fd4088e91c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:15 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:15.253+0000 7fd4088e91c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:15 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:15.339+0000 7fd4088e91c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:15 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:15.729+0000 7fd4088e91c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:56:16.825 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:16.524+0000 7fd4088e91c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:56:16.825 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:16.825+0000 7fd4088e91c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:56:17.397 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:17.145+0000 7fd4088e91c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:56:17.397 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:17.345+0000 7fd4088e91c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:56:17.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: Activating manager daemon y 2024-04-05T02:56:17.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: mgrmap e2: y(active, starting, since 0.00378693s) 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: Manager daemon y is now available 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' 2024-04-05T02:56:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' 2024-04-05T02:56:17.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:17 smithi033 ceph-mon[29244]: from='mgr.14100 172.21.15.33:0/1286511914' entity='mgr.y' 2024-04-05T02:56:18.004 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:18.004 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:56:18.004 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsid": "f4bb5524-f2f7-11ee-b649-cb9ed24678a4", 2024-04-05T02:56:18.004 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:56:18.004 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:56:18.004 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:56:18.004 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:18.005 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:56:18.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:18.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:56:18.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:56:18.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:56:18.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:56:18.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:56:18.006 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:56:18.009 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:18.009 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:56:18.009 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:56:18.010 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:18.011 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:56:18.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:18.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:56:04.452703+0000", 2024-04-05T02:56:18.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:18.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:18.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:56:18.012 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:56:18.012 INFO:teuthology.orchestra.run.smithi033.stdout:mgr not available, waiting (3/15)... 2024-04-05T02:56:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:18 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2024814317' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:56:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:18 smithi033 ceph-mon[29244]: mgrmap e3: y(active, since 1.00731s) 2024-04-05T02:56:20.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:20 smithi033 ceph-mon[29244]: mgrmap e4: y(active, since 2s) 2024-04-05T02:56:23.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:23 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/118099340' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:56:29.712 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:29.712 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:56:29.712 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsid": "f4bb5524-f2f7-11ee-b649-cb9ed24678a4", 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-04-05T02:56:29.713 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:56:29.714 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:56:29.717 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:29.718 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:56:29.719 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:56:04.452703+0000", 2024-04-05T02:56:29.720 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:56:29.720 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:56:29.720 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:56:29.720 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:56:29.720 INFO:teuthology.orchestra.run.smithi033.stdout:mgr is available 2024-04-05T02:56:31.193 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:30 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1851382300' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-05T02:56:31.196 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:31.197 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [global] 2024-04-05T02:56:31.197 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout fsid = f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:31.197 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [osd] 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-05T02:56:31.198 INFO:teuthology.orchestra.run.smithi033.stdout:Enabling cephadm module... 2024-04-05T02:56:32.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:31 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3789698170' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-05T02:56:32.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:31 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setuser ceph since I am not root 2024-04-05T02:56:32.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:31 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setgroup ceph since I am not root 2024-04-05T02:56:32.658 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:32.444+0000 7f856a1611c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:56:32.658 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:32.540+0000 7f856a1611c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:56:33.191 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:32 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3789698170' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-05T02:56:33.191 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:32 smithi033 ceph-mon[29244]: mgrmap e5: y(active, since 14s) 2024-04-05T02:56:33.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:33.192+0000 7f856a1611c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:56:33.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:33.282+0000 7f856a1611c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:56:33.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:33.368+0000 7f856a1611c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for the mgr to restart... 2024-04-05T02:56:33.702 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr epoch 5... 2024-04-05T02:56:33.782 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:33.519+0000 7f856a1611c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:56:33.782 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:33.601+0000 7f856a1611c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:56:33.783 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:33.783+0000 7f856a1611c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:56:34.052 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/651262517' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-05T02:56:34.053 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:33 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:33.967+0000 7f856a1611c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:56:34.053 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:34.053+0000 7f856a1611c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:56:34.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:34.149+0000 7f856a1611c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:56:34.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:34.235+0000 7f856a1611c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:56:35.620 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:35 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:35.293+0000 7f856a1611c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:56:35.620 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:35 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:35.424+0000 7f856a1611c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:56:35.620 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:35 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:35.618+0000 7f856a1611c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:56:35.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:35 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:35.846+0000 7f856a1611c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:56:36.464 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:36 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:36.022+0000 7f856a1611c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:56:36.464 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:36 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:36.203+0000 7f856a1611c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:56:36.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:36 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:36.464+0000 7f856a1611c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:56:36.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:36 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:36.550+0000 7f856a1611c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:56:37.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:36 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:36.941+0000 7f856a1611c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:56:38.029 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:37.730+0000 7f856a1611c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:56:38.030 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:38.029+0000 7f856a1611c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:56:38.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:38.345+0000 7f856a1611c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:56:38.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:38.539+0000 7f856a1611c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: Active manager daemon y restarted 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: Activating manager daemon y 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: osdmap e2: 0 total, 0 up, 0 in 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: mgrmap e6: y(active, starting, since 0.00318687s) 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: Manager daemon y is now available 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:38.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:38.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:56:38.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:38 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:56:39.897 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:39 smithi033 ceph-mon[29244]: Found migration_current of "None". Setting to last migration. 2024-04-05T02:56:39.897 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:39 smithi033 ceph-mon[29244]: mgrmap e7: y(active, since 1.00497s) 2024-04-05T02:56:40.175 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:56:40.175 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-05T02:56:40.175 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-05T02:56:40.175 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:56:40.175 INFO:teuthology.orchestra.run.smithi033.stdout:mgr epoch 5 is available 2024-04-05T02:56:40.175 INFO:teuthology.orchestra.run.smithi033.stdout:Setting orchestrator backend to cephadm... 2024-04-05T02:56:40.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:40 smithi033 ceph-mon[29244]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-05T02:56:40.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:40 smithi033 ceph-mon[29244]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-05T02:56:41.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:41 smithi033 ceph-mon[29244]: mgrmap e8: y(active, since 2s) 2024-04-05T02:56:41.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:41 smithi033 ceph-mon[29244]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:41.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:41 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:41.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:41 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:41.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:41 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:41.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:41 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:42.513 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-05T02:56:42.514 INFO:teuthology.orchestra.run.smithi033.stdout:Generating ssh key... 2024-04-05T02:56:42.897 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:42 smithi033 ceph-mon[29244]: [05/Apr/2024:02:56:40] ENGINE Bus STARTING 2024-04-05T02:56:42.897 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:42 smithi033 ceph-mon[29244]: [05/Apr/2024:02:56:41] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T02:56:42.897 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:42 smithi033 ceph-mon[29244]: [05/Apr/2024:02:56:41] ENGINE Bus STARTED 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: Generating public/private rsa key pair. 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: Your identification has been saved in /tmp/tmpujophv_7/key. 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: Your public key has been saved in /tmp/tmpujophv_7/key.pub. 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: The key fingerprint is: 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: SHA256:D2sgCghWY2TMbqJsl4olvulKXzqEDbFL0DnjmJNH5Ms ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: The key's randomart image is: 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: +---[RSA 3072]----+ 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: | o=B | 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |.oXo. | 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |oB=o | 2024-04-05T02:56:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |O*o+ | 2024-04-05T02:56:43.482 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |*oE o . S | 2024-04-05T02:56:43.482 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |o*.* . . + | 2024-04-05T02:56:43.482 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |++= . o . | 2024-04-05T02:56:43.482 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |ooo.o . | 2024-04-05T02:56:43.482 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: |++.o. | 2024-04-05T02:56:43.482 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: +----[SHA256]-----+ 2024-04-05T02:56:43.891 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-mon[29244]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:43.891 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:43.891 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:43 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:44.845 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyZJ437ILyy5exaR5p+zV6fAT9t3vn/4m0p9g+A8+brE67/bYFFniJTCl9eno29MRSONQOguLjd8V5EIh72xHcHQkUGo8Gm+kZfPlXdSFFzuzv5utz4yW8+cyWX6JKkAVMDl1kYmuJQSfzA+GHkP4PvJfB8z+iIjx/y0d61zkRMaE1dfPUAr9bQHBacxvOWI53xZ29Qp3jm4d5SRtEr63jtbBHixdGnXdUNvU/cJke6uyDNYKtMJfmPiYdNrEE/6PT7FrvVvQe5XSb1maJmYvVG5GZfpBnxeBARgHqqWDReerp6BFvp4qYWuEDNdMit2/hm7k6K0dHKIx22adF7nhu3hvIRxP9CkXuWBdxNpdJ2rAUMENfVt4dtU10hZq6AG7jBXXkM/VBgK+kZYBKx2XAtyjaZ9crPzNVga+G+mZ+VDx75s9LWsp5cGwZi1htbCbImKswNf1kdyP4G3IQY+fl5gSXEeFlqeo/sgXBRIth9Iq3Hr38Bu5qy8mx9k8hrsc= ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:56:44.845 INFO:teuthology.orchestra.run.smithi033.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-05T02:56:44.845 INFO:teuthology.orchestra.run.smithi033.stdout:Adding key to root@localhost authorized_keys... 2024-04-05T02:56:44.845 INFO:teuthology.orchestra.run.smithi033.stdout:Adding host smithi033... 2024-04-05T02:56:44.894 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:44 smithi033 ceph-mon[29244]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:44.894 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:44 smithi033 ceph-mon[29244]: Generating ssh key... 2024-04-05T02:56:45.846 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:45 smithi033 ceph-mon[29244]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:46.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:46 smithi033 ceph-mon[29244]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi033", "addr": "172.21.15.33", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:47.891 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:47 smithi033 ceph-mon[29244]: Deploying cephadm binary to smithi033 2024-04-05T02:56:47.891 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:47 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:47.891 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:47 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:48.161 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Added host 'smithi033' with addr '172.21.15.33' 2024-04-05T02:56:48.161 INFO:teuthology.orchestra.run.smithi033.stdout:Deploying unmanaged mon service... 2024-04-05T02:56:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:48 smithi033 ceph-mon[29244]: Added host smithi033 2024-04-05T02:56:49.429 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-05T02:56:49.429 INFO:teuthology.orchestra.run.smithi033.stdout:Deploying unmanaged mgr service... 2024-04-05T02:56:49.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:49 smithi033 ceph-mon[29244]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:49.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:49 smithi033 ceph-mon[29244]: Saving service mon spec with placement count:5 2024-04-05T02:56:49.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:49 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:51.016 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-05T02:56:51.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:51 smithi033 ceph-mon[29244]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:51.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:51 smithi033 ceph-mon[29244]: Saving service mgr spec with placement count:2 2024-04-05T02:56:51.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:51 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:52.800 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:52 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/397989095' entity='client.admin' 2024-04-05T02:56:52.800 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:52 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:52.800 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:52 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:54.043 INFO:teuthology.orchestra.run.smithi033.stdout:Enabling the dashboard module... 2024-04-05T02:56:54.079 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:54 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/123062765' entity='client.admin' 2024-04-05T02:56:54.079 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:54 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:55.804 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:55 smithi033 ceph-mon[29244]: from='mgr.14120 172.21.15.33:0/884171871' entity='mgr.y' 2024-04-05T02:56:55.805 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:55 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1300701029' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-05T02:56:55.805 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setuser ceph since I am not root 2024-04-05T02:56:55.805 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setgroup ceph since I am not root 2024-04-05T02:56:56.388 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:56 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:56.105+0000 7f2b583741c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:56:56.388 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:56 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:56.198+0000 7f2b583741c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:56:56.700 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:56 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1300701029' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-05T02:56:56.700 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:56 smithi033 ceph-mon[29244]: mgrmap e9: y(active, since 17s) 2024-04-05T02:56:56.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:56 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:56.845+0000 7f2b583741c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:56:56.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:56 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:56.936+0000 7f2b583741c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:56:57.403 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.022+0000 7f2b583741c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:56:57.403 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.176+0000 7f2b583741c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:56:57.403 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.258+0000 7f2b583741c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:56:57.700 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/173796466' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-05T02:56:57.700 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.432+0000 7f2b583741c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:56:57.700 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.614+0000 7f2b583741c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:56:57.700 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.700+0000 7f2b583741c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for the mgr to restart... 2024-04-05T02:56:57.707 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr epoch 9... 2024-04-05T02:56:57.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.798+0000 7f2b583741c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:56:57.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:57.885+0000 7f2b583741c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:56:59.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:58.947+0000 7f2b583741c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:56:59.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:59.078+0000 7f2b583741c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:56:59.677 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:59.272+0000 7f2b583741c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:56:59.677 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:59.500+0000 7f2b583741c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:56:59.677 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:59.676+0000 7f2b583741c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:56:59.954 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:56:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:56:59.859+0000 7f2b583741c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:57:00.209 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:57:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:57:00.127+0000 7f2b583741c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:57:00.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:57:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:57:00.218+0000 7f2b583741c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:57:00.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:57:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:57:00.623+0000 7f2b583741c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:57:01.719 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:57:01 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:57:01.421+0000 7f2b583741c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:57:01.719 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:57:01 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:57:01.719+0000 7f2b583741c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:57:02.290 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:57:02.033+0000 7f2b583741c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:57:02.290 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T02:57:02.229+0000 7f2b583741c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:57:02.290 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: Active manager daemon y restarted 2024-04-05T02:57:02.290 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: Activating manager daemon y 2024-04-05T02:57:02.290 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: osdmap e3: 0 total, 0 up, 0 in 2024-04-05T02:57:02.290 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: mgrmap e10: y(active, starting, since 0.00538875s) 2024-04-05T02:57:02.290 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: Manager daemon y is now available 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:57:02.291 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:57:03.834 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:57:03.834 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-05T02:57:03.834 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-05T02:57:03.834 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:57:03.834 INFO:teuthology.orchestra.run.smithi033.stdout:mgr epoch 9 is available 2024-04-05T02:57:03.834 INFO:teuthology.orchestra.run.smithi033.stdout:Generating a dashboard self-signed certificate... 2024-04-05T02:57:04.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:04 smithi033 ceph-mon[29244]: mgrmap e11: y(active, since 1.0058s) 2024-04-05T02:57:04.676 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:04 smithi033 ceph-mon[29244]: [05/Apr/2024:02:57:03] ENGINE Bus STARTING 2024-04-05T02:57:04.676 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:04.676 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:04 smithi033 ceph-mon[29244]: [05/Apr/2024:02:57:03] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T02:57:04.676 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:04 smithi033 ceph-mon[29244]: [05/Apr/2024:02:57:03] ENGINE Bus STARTED 2024-04-05T02:57:04.676 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:05.349 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:05 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:05.349 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:05 smithi033 ceph-mon[29244]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:05.349 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:05 smithi033 ceph-mon[29244]: mgrmap e12: y(active, since 2s) 2024-04-05T02:57:05.349 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:05 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:05.349 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:05 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:05.351 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-05T02:57:05.351 INFO:teuthology.orchestra.run.smithi033.stdout:Creating initial admin user... 2024-04-05T02:57:07.538 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$o6lmDluL1aR6EqqwW.4Lt.GaAkYKIupQ542IK4mpxHpQw2o35mXMO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712285826, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-05T02:57:07.538 INFO:teuthology.orchestra.run.smithi033.stdout:Fetching dashboard port number... 2024-04-05T02:57:08.132 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:07 smithi033 ceph-mon[29244]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:08.132 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:07 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:08.663 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 8443 2024-04-05T02:57:08.673 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:57:08.673 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-04-05T02:57:08.681 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:57:08.681 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-04-05T02:57:08.681 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-04-05T02:57:08.681 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-05T02:57:08.682 INFO:teuthology.orchestra.run.smithi033.stdout:Ceph Dashboard is now available at: 2024-04-05T02:57:08.683 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:08.683 INFO:teuthology.orchestra.run.smithi033.stdout: URL: https://smithi033.front.sepia.ceph.com:8443/ 2024-04-05T02:57:08.683 INFO:teuthology.orchestra.run.smithi033.stdout: User: admin 2024-04-05T02:57:08.683 INFO:teuthology.orchestra.run.smithi033.stdout: Password: 40zfx4zi9b 2024-04-05T02:57:08.683 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:08.683 INFO:teuthology.orchestra.run.smithi033.stdout:Saving cluster configuration to /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config directory 2024-04-05T02:57:08.683 INFO:teuthology.orchestra.run.smithi033.stdout:Enabling autotune for osd_memory_target 2024-04-05T02:57:09.071 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:08 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/972341914' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-05T02:57:10.136 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:09 smithi033 ceph-mon[29244]: mgrmap e13: y(active, since 6s) 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout:Or, if you are only running a single cluster on this host: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.101 INFO:teuthology.orchestra.run.smithi033.stdout: ceph telemetry on 2024-04-05T02:57:11.102 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.102 INFO:teuthology.orchestra.run.smithi033.stdout:For more information see: 2024-04-05T02:57:11.102 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.102 INFO:teuthology.orchestra.run.smithi033.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-05T02:57:11.102 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:11.102 INFO:teuthology.orchestra.run.smithi033.stdout:Bootstrap complete. 2024-04-05T02:57:11.139 INFO:tasks.cephadm:Fetching config... 2024-04-05T02:57:11.139 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:57:11.139 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-05T02:57:11.155 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-05T02:57:11.156 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:57:11.156 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-05T02:57:11.211 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-05T02:57:11.214 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:57:11.214 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-05T02:57:11.280 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-05T02:57:11.280 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:57:11.281 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-05T02:57:11.335 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-05T02:57:11.336 DEBUG:teuthology.orchestra.run.smithi033:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyZJ437ILyy5exaR5p+zV6fAT9t3vn/4m0p9g+A8+brE67/bYFFniJTCl9eno29MRSONQOguLjd8V5EIh72xHcHQkUGo8Gm+kZfPlXdSFFzuzv5utz4yW8+cyWX6JKkAVMDl1kYmuJQSfzA+GHkP4PvJfB8z+iIjx/y0d61zkRMaE1dfPUAr9bQHBacxvOWI53xZ29Qp3jm4d5SRtEr63jtbBHixdGnXdUNvU/cJke6uyDNYKtMJfmPiYdNrEE/6PT7FrvVvQe5XSb1maJmYvVG5GZfpBnxeBARgHqqWDReerp6BFvp4qYWuEDNdMit2/hm7k6K0dHKIx22adF7nhu3hvIRxP9CkXuWBdxNpdJ2rAUMENfVt4dtU10hZq6AG7jBXXkM/VBgK+kZYBKx2XAtyjaZ9crPzNVga+G+mZ+VDx75s9LWsp5cGwZi1htbCbImKswNf1kdyP4G3IQY+fl5gSXEeFlqeo/sgXBRIth9Iq3Hr38Bu5qy8mx9k8hrsc= ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-05T02:57:11.418 INFO:teuthology.orchestra.run.smithi033.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyZJ437ILyy5exaR5p+zV6fAT9t3vn/4m0p9g+A8+brE67/bYFFniJTCl9eno29MRSONQOguLjd8V5EIh72xHcHQkUGo8Gm+kZfPlXdSFFzuzv5utz4yW8+cyWX6JKkAVMDl1kYmuJQSfzA+GHkP4PvJfB8z+iIjx/y0d61zkRMaE1dfPUAr9bQHBacxvOWI53xZ29Qp3jm4d5SRtEr63jtbBHixdGnXdUNvU/cJke6uyDNYKtMJfmPiYdNrEE/6PT7FrvVvQe5XSb1maJmYvVG5GZfpBnxeBARgHqqWDReerp6BFvp4qYWuEDNdMit2/hm7k6K0dHKIx22adF7nhu3hvIRxP9CkXuWBdxNpdJ2rAUMENfVt4dtU10hZq6AG7jBXXkM/VBgK+kZYBKx2XAtyjaZ9crPzNVga+G+mZ+VDx75s9LWsp5cGwZi1htbCbImKswNf1kdyP4G3IQY+fl5gSXEeFlqeo/sgXBRIth9Iq3Hr38Bu5qy8mx9k8hrsc= ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:57:11.433 DEBUG:teuthology.orchestra.run.smithi151:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyZJ437ILyy5exaR5p+zV6fAT9t3vn/4m0p9g+A8+brE67/bYFFniJTCl9eno29MRSONQOguLjd8V5EIh72xHcHQkUGo8Gm+kZfPlXdSFFzuzv5utz4yW8+cyWX6JKkAVMDl1kYmuJQSfzA+GHkP4PvJfB8z+iIjx/y0d61zkRMaE1dfPUAr9bQHBacxvOWI53xZ29Qp3jm4d5SRtEr63jtbBHixdGnXdUNvU/cJke6uyDNYKtMJfmPiYdNrEE/6PT7FrvVvQe5XSb1maJmYvVG5GZfpBnxeBARgHqqWDReerp6BFvp4qYWuEDNdMit2/hm7k6K0dHKIx22adF7nhu3hvIRxP9CkXuWBdxNpdJ2rAUMENfVt4dtU10hZq6AG7jBXXkM/VBgK+kZYBKx2XAtyjaZ9crPzNVga+G+mZ+VDx75s9LWsp5cGwZi1htbCbImKswNf1kdyP4G3IQY+fl5gSXEeFlqeo/sgXBRIth9Iq3Hr38Bu5qy8mx9k8hrsc= ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-05T02:57:11.478 INFO:teuthology.orchestra.run.smithi151.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyZJ437ILyy5exaR5p+zV6fAT9t3vn/4m0p9g+A8+brE67/bYFFniJTCl9eno29MRSONQOguLjd8V5EIh72xHcHQkUGo8Gm+kZfPlXdSFFzuzv5utz4yW8+cyWX6JKkAVMDl1kYmuJQSfzA+GHkP4PvJfB8z+iIjx/y0d61zkRMaE1dfPUAr9bQHBacxvOWI53xZ29Qp3jm4d5SRtEr63jtbBHixdGnXdUNvU/cJke6uyDNYKtMJfmPiYdNrEE/6PT7FrvVvQe5XSb1maJmYvVG5GZfpBnxeBARgHqqWDReerp6BFvp4qYWuEDNdMit2/hm7k6K0dHKIx22adF7nhu3hvIRxP9CkXuWBdxNpdJ2rAUMENfVt4dtU10hZq6AG7jBXXkM/VBgK+kZYBKx2XAtyjaZ9crPzNVga+G+mZ+VDx75s9LWsp5cGwZi1htbCbImKswNf1kdyP4G3IQY+fl5gSXEeFlqeo/sgXBRIth9Iq3Hr38Bu5qy8mx9k8hrsc= ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:57:11.494 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-05T02:57:11.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:11 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3995771212' entity='client.admin' 2024-04-05T02:57:11.754 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:15.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:15 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2708017030' entity='client.admin' 2024-04-05T02:57:15.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:15.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:15.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi033", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:57:15.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:15.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:15.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:15.797 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-05T02:57:15.797 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-05T02:57:16.055 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:18.467 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi151 2024-04-05T02:57:18.467 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:57:18.467 DEBUG:teuthology.orchestra.run.smithi151:> dd of=/etc/ceph/ceph.conf 2024-04-05T02:57:18.488 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:57:18.488 DEBUG:teuthology.orchestra.run.smithi151:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:57:18.545 INFO:tasks.cephadm:Adding host smithi151 to orchestrator... 2024-04-05T02:57:18.545 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch host add smithi151 2024-04-05T02:57:18.801 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:18.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:18 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:20.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:20 smithi033 ceph-mon[29244]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi151", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:21.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:21 smithi033 ceph-mon[29244]: Deploying cephadm binary to smithi151 2024-04-05T02:57:22.634 INFO:teuthology.orchestra.run.smithi033.stdout:Added host 'smithi151' with addr '172.21.15.151' 2024-04-05T02:57:23.282 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-05T02:57:23.542 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:23.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:23 smithi033 ceph-mon[29244]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:23.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:23 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:23.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:23 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:23.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:23 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:24.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:24 smithi033 ceph-mon[29244]: Added host smithi151 2024-04-05T02:57:24.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:24.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:25.199 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T02:57:25.199 INFO:teuthology.orchestra.run.smithi033.stdout:[{"addr": "172.21.15.33", "hostname": "smithi033", "labels": [], "status": ""}, {"addr": "172.21.15.151", "hostname": "smithi151", "labels": [], "status": ""}] 2024-04-05T02:57:25.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:25 smithi033 ceph-mon[29244]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:25.732 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-05T02:57:25.733 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-05T02:57:25.986 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:26.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:26 smithi033 ceph-mon[29244]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi151", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:27.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:27 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:28.298 INFO:teuthology.orchestra.run.smithi033.stderr:adjusted tunables profile to default 2024-04-05T02:57:28.587 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T02:57:28.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:28.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:57:28.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:57:28.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/205666339' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-05T02:57:28.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:28.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:28.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:28 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:28.855 INFO:tasks.cephadm:Adding mon.a on smithi033 2024-04-05T02:57:28.855 INFO:tasks.cephadm:Adding mon.c on smithi033 2024-04-05T02:57:28.855 INFO:tasks.cephadm:Adding mon.b on smithi151 2024-04-05T02:57:28.855 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch apply mon '3;smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi151:172.21.15.151=b' 2024-04-05T02:57:29.105 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:29.565 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:29 smithi033 ceph-mon[29244]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:29.566 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:29 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/205666339' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-05T02:57:29.566 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:29 smithi033 ceph-mon[29244]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:30.715 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled mon update... 2024-04-05T02:57:31.311 DEBUG:teuthology.orchestra.run.smithi033:mon.c> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.c.service 2024-04-05T02:57:31.314 DEBUG:teuthology.orchestra.run.smithi151:mon.b> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.b.service 2024-04-05T02:57:31.317 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-05T02:57:31.317 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-05T02:57:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:32.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:32 smithi033 ceph-mon[29244]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi151:172.21.15.151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:32 smithi033 ceph-mon[29244]: Saving service mon spec with placement smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi151:172.21.15.151=b;count:3 2024-04-05T02:57:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:32 smithi033 ceph-mon[29244]: Deploying daemon mon.b on smithi151 2024-04-05T02:57:33.140 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:33 smithi033 ceph-mon[29244]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:35.578 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:57:35.578 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":1,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","modified":"2024-04-05T02:56:01.729844Z","created":"2024-04-05T02:56:01.729844Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-05T02:57:35.580 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 1 2024-04-05T02:57:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:35 smithi033 ceph-mon[29244]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:36.139 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:35 smithi151 systemd[1]: Started Ceph mon.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T02:57:37.499 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-05T02:57:37.499 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-05T02:57:37.762 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:57:41.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: mon.a calling monitor election 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: mon.b calling monitor election 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:57:41.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: monmap e2: 2 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:41.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: fsmap 2024-04-05T02:57:41.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:41.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: mgrmap e13: y(active, since 38s) 2024-04-05T02:57:41.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: overall HEALTH_OK 2024-04-05T02:57:41.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:41.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:41.438 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:41.438 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: mon.a calling monitor election 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: mon.b calling monitor election 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:41.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: monmap e2: 2 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: fsmap 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: mgrmap e13: y(active, since 38s) 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: overall HEALTH_OK 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:41.440 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:42.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:42 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:42.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:42 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:42.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:42 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:42.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:42 smithi151 ceph-mon[32789]: Deploying daemon mon.c on smithi033 2024-04-05T02:57:42.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:42 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:42.303 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:57:42.303 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":2,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","modified":"2024-04-05T02:57:35.936273Z","created":"2024-04-05T02:56:01.729844Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-05T02:57:42.305 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 2 2024-04-05T02:57:42.344 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:42 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:42.344 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:42 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:42.344 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:42 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:42.344 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:42 smithi033 ceph-mon[29244]: Deploying daemon mon.c on smithi033 2024-04-05T02:57:42.344 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:42 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:43 smithi151 ceph-mon[32789]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/1774843502' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:57:43.342 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:43 smithi033 ceph-mon[29244]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:43.342 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/1774843502' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:57:43.895 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-05T02:57:43.895 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-05T02:57:44.154 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:57:44.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:44 smithi033 systemd[1]: Started Ceph mon.c for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: mon.a calling monitor election 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: mon.b calling monitor election 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: fsmap 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: mgrmap e13: y(active, since 47s) 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: mon.c (rank 2) addr [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] is down (out of quorum) 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: mon.a calling monitor election 2024-04-05T02:57:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: mon.b calling monitor election 2024-04-05T02:57:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: fsmap 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: mgrmap e13: y(active, since 47s) 2024-04-05T02:57:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-05T02:57:49.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-04-05T02:57:49.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-05T02:57:49.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: mon.c (rank 2) addr [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] is down (out of quorum) 2024-04-05T02:57:49.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:49.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:50.196 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-05T02:57:50.196 INFO:teuthology.orchestra.run.smithi151.stdout:{"epoch":3,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","modified":"2024-04-05T02:57:44.232912Z","created":"2024-04-05T02:56:01.729844Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:3300","nonce":0},{"type":"v1","addr":"172.21.15.151:6789","nonce":0}]},"addr":"172.21.15.151:6789/0","public_addr":"172.21.15.151:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3301","nonce":0},{"type":"v1","addr":"172.21.15.33:6790","nonce":0}]},"addr":"172.21.15.33:6790/0","public_addr":"172.21.15.33:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-05T02:57:50.199 INFO:teuthology.orchestra.run.smithi151.stderr:dumped monmap epoch 3 2024-04-05T02:57:50.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:50.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:50.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:50.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:50 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/3390867225' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:57:50.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:50.766 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:50.766 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:50 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/3390867225' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:57:50.767 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.113 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-05T02:57:51.113 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.a calling monitor election 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.b calling monitor election 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.393 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: fsmap 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mgrmap e13: y(active, since 47s) 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.c (rank 2) addr [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] is down (out of quorum) 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:51.394 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:51.395 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:51.395 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:51.395 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:51.395 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/3390867225' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:57:51.395 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:51.916 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:52.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: mon.c calling monitor election 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: Reconfiguring mon.a (unknown last config time)... 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: Reconfiguring daemon mon.a on smithi033 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: mon.c calling monitor election 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: mon.b calling monitor election 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: mon.a calling monitor election 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: fsmap 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: mgrmap e13: y(active, since 49s) 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T02:57:52.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: Cluster is now healthy 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[29244]: overall HEALTH_OK 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.c calling monitor election 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Reconfiguring mon.a (unknown last config time)... 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Reconfiguring daemon mon.a on smithi033 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.c calling monitor election 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.b calling monitor election 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.a calling monitor election 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: fsmap 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:52.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: mgrmap e13: y(active, since 49s) 2024-04-05T02:57:52.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T02:57:52.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: Cluster is now healthy 2024-04-05T02:57:52.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:51 smithi033 ceph-mon[40008]: overall HEALTH_OK 2024-04-05T02:57:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: mon.c calling monitor election 2024-04-05T02:57:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:57:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: Reconfiguring mon.a (unknown last config time)... 2024-04-05T02:57:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: Reconfiguring daemon mon.a on smithi033 2024-04-05T02:57:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: mon.c calling monitor election 2024-04-05T02:57:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: mon.b calling monitor election 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: mon.a calling monitor election 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: fsmap 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: mgrmap e13: y(active, since 49s) 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: Cluster is now healthy 2024-04-05T02:57:52.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:51 smithi151 ceph-mon[32789]: overall HEALTH_OK 2024-04-05T02:57:53.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:53.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[29244]: Reconfiguring daemon mon.c on smithi033 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:53.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:52 smithi033 ceph-mon[40008]: Reconfiguring daemon mon.c on smithi033 2024-04-05T02:57:53.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:53.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:53.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:53.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:57:53.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:53.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:53.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:53.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:52 smithi151 ceph-mon[32789]: Reconfiguring daemon mon.c on smithi033 2024-04-05T02:57:54.046 INFO:teuthology.orchestra.run.smithi033.stdout:# minimal ceph.conf for f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:57:54.046 INFO:teuthology.orchestra.run.smithi033.stdout:[global] 2024-04-05T02:57:54.046 INFO:teuthology.orchestra.run.smithi033.stdout: fsid = f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T02:57:54.046 INFO:teuthology.orchestra.run.smithi033.stdout: mon_host = [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] 2024-04-05T02:57:54.240 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:54 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3608175498' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:54.241 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:54 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3608175498' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:54.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:54 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3608175498' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:54.971 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-05T02:57:54.971 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:57:54.971 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.conf 2024-04-05T02:57:55.000 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:57:55.001 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:57:55.067 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:57:55.068 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.conf 2024-04-05T02:57:55.104 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:57:55.104 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:57:55.174 INFO:tasks.cephadm:Adding mgr.y on smithi033 2024-04-05T02:57:55.174 INFO:tasks.cephadm:Adding mgr.x on smithi151 2024-04-05T02:57:55.174 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch apply mgr '2;smithi033=y;smithi151=x' 2024-04-05T02:57:55.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:55 smithi033 ceph-mon[29244]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:55.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:55 smithi033 ceph-mon[40008]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:55.463 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:57:55.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:55 smithi151 ceph-mon[32789]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:56 smithi151 ceph-mon[32789]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:57:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:56.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:56.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:56 smithi151 ceph-mon[32789]: Reconfiguring daemon mon.b on smithi151 2024-04-05T02:57:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[29244]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[29244]: Reconfiguring daemon mon.b on smithi151 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[40008]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:57:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:56.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:56.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:56.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:56 smithi033 ceph-mon[40008]: Reconfiguring daemon mon.b on smithi151 2024-04-05T02:57:57.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:57 smithi151 ceph-mon[32789]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:57.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:57 smithi033 ceph-mon[29244]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:57 smithi033 ceph-mon[40008]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:58.119 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled mgr update... 2024-04-05T02:57:58.735 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi033=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: Saving service mgr spec with placement smithi033=y;smithi151=x;count:2 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.736 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:57:58.737 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-05T02:57:58.737 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:57:58.737 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.737 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: Deploying daemon mgr.x on smithi151 2024-04-05T02:57:58.737 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:57:58 smithi151 ceph-mon[32789]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:58.763 DEBUG:teuthology.orchestra.run.smithi151:mgr.x> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service 2024-04-05T02:57:58.765 INFO:tasks.cephadm:Deploying OSDs... 2024-04-05T02:57:58.765 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T02:57:58.765 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/scratch_devs of=/dev/stdout 2024-04-05T02:57:58.781 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-05T02:57:58.781 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_1 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-04-05 02:57:43.670845334 +0000 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-04-05 02:45:34.775515859 +0000 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-04-05 02:45:34.775515859 +0000 2024-04-05T02:57:58.836 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-04-05 02:45:34.775515859 +0000 2024-04-05T02:57:58.837 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-05T02:57:58.904 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-04-05T02:57:58.905 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-04-05T02:57:58.905 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000206576 s, 2.5 MB/s 2024-04-05T02:57:58.906 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-05T02:57:58.961 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_2 2024-04-05T02:57:58.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi033=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: Saving service mgr spec with placement smithi033=y;smithi151=x;count:2 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: Deploying daemon mgr.x on smithi151 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[29244]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:58.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi033=y;smithi151=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: Saving service mgr spec with placement smithi033=y;smithi151=x;count:2 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: Deploying daemon mgr.x on smithi151 2024-04-05T02:57:58.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:57:58 smithi033 ceph-mon[40008]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:57:58.997 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-05T02:57:58.997 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:57:58.997 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-05T02:57:58.997 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:57:58.997 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:57:58.998 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-04-05 02:57:43.670845334 +0000 2024-04-05T02:57:58.998 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-04-05 02:45:35.113510423 +0000 2024-04-05T02:57:58.998 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-04-05 02:45:35.113510423 +0000 2024-04-05T02:57:58.998 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-04-05 02:45:35.113510423 +0000 2024-04-05T02:57:58.998 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-05T02:57:59.062 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-04-05T02:57:59.063 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-04-05T02:57:59.063 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000173972 s, 2.9 MB/s 2024-04-05T02:57:59.064 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-05T02:57:59.119 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_3 2024-04-05T02:57:59.173 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-04-05 02:57:43.671845318 +0000 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-04-05 02:45:35.467504729 +0000 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-04-05 02:45:35.467504729 +0000 2024-04-05T02:57:59.174 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-04-05 02:45:35.467504729 +0000 2024-04-05T02:57:59.175 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-05T02:57:59.238 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-04-05T02:57:59.239 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-04-05T02:57:59.239 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000247358 s, 2.1 MB/s 2024-04-05T02:57:59.240 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-05T02:57:59.295 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_4 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-04-05 02:57:43.671845318 +0000 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-04-05 02:45:35.810499212 +0000 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-04-05 02:45:35.810499212 +0000 2024-04-05T02:57:59.349 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-04-05 02:45:35.810499212 +0000 2024-04-05T02:57:59.349 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-05T02:57:59.414 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-04-05T02:57:59.414 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-04-05T02:57:59.414 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000181455 s, 2.8 MB/s 2024-04-05T02:57:59.415 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-05T02:57:59.470 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T02:57:59.471 DEBUG:teuthology.orchestra.run.smithi151:> dd if=/scratch_devs of=/dev/stdout 2024-04-05T02:57:59.486 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-05T02:57:59.487 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_1 2024-04-05T02:57:59.541 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-05T02:57:59.541 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:57:59.541 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-04-05T02:57:59.541 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:57:59.541 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:57:59.542 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-05 02:57:35.021759477 +0000 2024-04-05T02:57:59.542 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-05 02:45:10.451728645 +0000 2024-04-05T02:57:59.542 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-05 02:45:10.451728645 +0000 2024-04-05T02:57:59.542 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-05 02:45:10.451728645 +0000 2024-04-05T02:57:59.542 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-05T02:57:59.608 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-05T02:57:59.608 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-05T02:57:59.608 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000243407 s, 2.1 MB/s 2024-04-05T02:57:59.610 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-05T02:57:59.665 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_2 2024-04-05T02:57:59.720 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-05 02:57:35.021759477 +0000 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-05 02:45:10.794725073 +0000 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-05 02:45:10.794725073 +0000 2024-04-05T02:57:59.721 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-05 02:45:10.794725073 +0000 2024-04-05T02:57:59.721 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-05T02:57:59.845 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-05T02:57:59.845 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-05T02:57:59.845 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000214296 s, 2.4 MB/s 2024-04-05T02:57:59.846 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-05T02:57:59.862 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_3 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-05 02:57:59.799483283 +0000 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-05 02:45:11.132721554 +0000 2024-04-05T02:57:59.922 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-05 02:45:11.132721554 +0000 2024-04-05T02:57:59.923 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-05 02:45:11.132721554 +0000 2024-04-05T02:57:59.923 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-05T02:58:00.027 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-05T02:58:00.027 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-05T02:58:00.027 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000196777 s, 2.6 MB/s 2024-04-05T02:58:00.028 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-05T02:58:00.043 DEBUG:teuthology.orchestra.run.smithi151:> stat /dev/vg_nvme/lv_4 2024-04-05T02:58:00.099 INFO:teuthology.orchestra.run.smithi151.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout:Access: 2024-04-05 02:57:59.983481204 +0000 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout:Modify: 2024-04-05 02:45:11.464718097 +0000 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout:Change: 2024-04-05 02:45:11.464718097 +0000 2024-04-05T02:58:00.100 INFO:teuthology.orchestra.run.smithi151.stdout: Birth: 2024-04-05 02:45:11.464718097 +0000 2024-04-05T02:58:00.100 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-05T02:58:00.166 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 systemd[1]: Starting Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T02:58:00.169 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records in 2024-04-05T02:58:00.169 INFO:teuthology.orchestra.run.smithi151.stderr:1+0 records out 2024-04-05T02:58:00.169 INFO:teuthology.orchestra.run.smithi151.stderr:512 bytes copied, 0.000179977 s, 2.8 MB/s 2024-04-05T02:58:00.171 DEBUG:teuthology.orchestra.run.smithi151:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-05T02:58:00.229 INFO:tasks.cephadm:Deploying osd.0 on smithi033 with /dev/vg_nvme/lv_4... 2024-04-05T02:58:00.229 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-05T02:58:00.461 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 podman[34623]: 2024-04-05 02:58:00.260546602 +0000 UTC m=+0.106734738 container create c743b76f858720fe2d79c98d146d45bc1a6f64f261700760f4f27901221946d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-05T02:58:00.462 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 podman[34623]: 2024-04-05 02:58:00.172810819 +0000 UTC m=+0.018998971 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T02:58:00.462 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 podman[34623]: 2024-04-05 02:58:00.462529125 +0000 UTC m=+0.308717260 container init c743b76f858720fe2d79c98d146d45bc1a6f64f261700760f4f27901221946d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-05T02:58:00.495 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:58:00.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 podman[34623]: 2024-04-05 02:58:00.465174211 +0000 UTC m=+0.311362350 container start c743b76f858720fe2d79c98d146d45bc1a6f64f261700760f4f27901221946d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T02:58:00.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 ceph-mgr[34672]: -- 172.21.15.151:0/1950628683 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b154b949c0 con 0x55b154b6a800 2024-04-05T02:58:00.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 bash[34623]: c743b76f858720fe2d79c98d146d45bc1a6f64f261700760f4f27901221946d9 2024-04-05T02:58:00.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 systemd[1]: Started Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T02:58:01.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:00.996+0000 7f6a5bc701c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:58:01.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:01.100+0000 7f6a5bc701c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:58:01.465 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[29244]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:01.465 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[40008]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:01 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-mon[32789]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:02.030 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:01.752+0000 7f6a5bc701c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:58:02.030 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:01.841+0000 7f6a5bc701c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:58:02.030 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:01.926+0000 7f6a5bc701c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:58:02.257 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-05T02:58:02.258 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-05T02:58:02.258 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-04-05T02:58:02.258 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-04-05T02:58:02.258 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0149719 s, 700 MB/s 2024-04-05T02:58:02.258 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-04-05T02:58:02.495 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:02.074+0000 7f6a5bc701c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:58:02.496 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:02.152+0000 7f6a5bc701c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:58:02.496 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:02.318+0000 7f6a5bc701c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:58:02.496 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:02.495+0000 7f6a5bc701c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:58:02.750 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:02.577+0000 7f6a5bc701c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:58:02.750 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:02.668+0000 7f6a5bc701c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:58:02.751 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:02.750+0000 7f6a5bc701c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: Reconfiguring mgr.y (unknown last config time)... 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: Reconfiguring daemon mgr.y on smithi033 2024-04-05T02:58:02.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: Reconfiguring mgr.y (unknown last config time)... 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:58:02.889 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:02.890 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: Reconfiguring daemon mgr.y on smithi033 2024-04-05T02:58:02.890 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:02.890 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:02.891 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi033:vg_nvme/lv_4 2024-04-05T02:58:03.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:03.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:03.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:03.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:03.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:03.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: Reconfiguring mgr.y (unknown last config time)... 2024-04-05T02:58:03.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:58:03.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:58:03.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:03.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: Reconfiguring daemon mgr.y on smithi033 2024-04-05T02:58:03.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:03.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:03.880 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:58:04.125 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:03 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:03.837+0000 7f6a5bc701c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:58:04.125 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:03 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:03.969+0000 7f6a5bc701c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:58:04.395 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:04.166+0000 7f6a5bc701c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:58:04.395 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:04.395+0000 7f6a5bc701c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:58:04.746 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:04.567+0000 7f6a5bc701c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:58:04.746 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:04.746+0000 7f6a5bc701c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:58:05.006 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.006 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.006 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:05.006 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:05.006 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:05.006 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.006 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:04 smithi151 ceph-mon[32789]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[40008]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.058 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.059 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:05.059 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:05.059 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:05.059 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:05.059 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:04 smithi033 ceph-mon[29244]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:05.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:05.006+0000 7f6a5bc701c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:58:05.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:05.091+0000 7f6a5bc701c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:58:05.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:05.508+0000 7f6a5bc701c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:58:06.113 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:05 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:06.113 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:05 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:06.113 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:05 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:06.113 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:05 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:06.114 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:05 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:06.114 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:05 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:05 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:05 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:05 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:06.570 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:06 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:06.278+0000 7f6a5bc701c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:58:06.571 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:06 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:06.570+0000 7f6a5bc701c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:58:07.175 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:06 smithi033 ceph-mon[29244]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:07.175 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:06 smithi033 ceph-mon[29244]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:07.176 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:06 smithi033 ceph-mon[40008]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:07.176 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:06 smithi033 ceph-mon[40008]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:06 smithi151 ceph-mon[32789]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:06 smithi151 ceph-mon[32789]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:07.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:06 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:06.887+0000 7f6a5bc701c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:58:07.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 02:58:07 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T02:58:07.077+0000 7f6a5bc701c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[29244]: Standby manager daemon x started 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[40008]: Standby manager daemon x started 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:58:08.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:07 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:58:08.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:07 smithi151 ceph-mon[32789]: Standby manager daemon x started 2024-04-05T02:58:08.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:07 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:58:08.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:07 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:58:08.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:07 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:58:08.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:07 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/358648028' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:58:09.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/136042131' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7102dab8-6fe6-43ae-b43a-8c063343eaf0"}]: dispatch 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/136042131' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7102dab8-6fe6-43ae-b43a-8c063343eaf0"}]': finished 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: osdmap e5: 1 total, 0 up, 1 in 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1519116767' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/136042131' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7102dab8-6fe6-43ae-b43a-8c063343eaf0"}]: dispatch 2024-04-05T02:58:09.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/136042131' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7102dab8-6fe6-43ae-b43a-8c063343eaf0"}]': finished 2024-04-05T02:58:09.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: osdmap e5: 1 total, 0 up, 1 in 2024-04-05T02:58:09.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:09.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:09.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:08 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1519116767' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/136042131' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7102dab8-6fe6-43ae-b43a-8c063343eaf0"}]: dispatch 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/136042131' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7102dab8-6fe6-43ae-b43a-8c063343eaf0"}]': finished 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: osdmap e5: 1 total, 0 up, 1 in 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:08 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1519116767' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:11.560 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:11 smithi033 ceph-mon[29244]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:11.561 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:11 smithi033 ceph-mon[40008]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:11.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:11 smithi151 ceph-mon[32789]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:13.554 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:13 smithi033 ceph-mon[29244]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:13.554 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:13 smithi033 ceph-mon[40008]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:13.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:13 smithi151 ceph-mon[32789]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:14.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:14 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:58:14.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:14 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:14.419 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:14 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:58:14.419 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:14 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:14 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:58:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:14 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:15.555 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:15 smithi033 ceph-mon[29244]: Deploying daemon osd.0 on smithi033 2024-04-05T02:58:15.555 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:15 smithi033 ceph-mon[29244]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:15.555 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:15 smithi033 ceph-mon[40008]: Deploying daemon osd.0 on smithi033 2024-04-05T02:58:15.555 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:15 smithi033 ceph-mon[40008]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:15.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:15 smithi151 ceph-mon[32789]: Deploying daemon osd.0 on smithi033 2024-04-05T02:58:15.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:15 smithi151 ceph-mon[32789]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:17.437 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:17 smithi033 ceph-mon[29244]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:17.438 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:17 smithi033 ceph-mon[40008]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:17 smithi151 ceph-mon[32789]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:19.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:19 smithi033 ceph-mon[29244]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:19.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:19.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:19.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:19 smithi033 ceph-mon[40008]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:19.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:19.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:19 smithi151 ceph-mon[32789]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:20.595 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:20.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:21.559 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:21 smithi033 ceph-mon[40008]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:21.559 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:21 smithi033 ceph-mon[29244]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:21.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:21 smithi151 ceph-mon[32789]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:22.621 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 0 on host 'smithi033' 2024-04-05T02:58:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:23.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:58:23.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:23.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:23.222 DEBUG:teuthology.orchestra.run.smithi033:osd.0> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.0.service 2024-04-05T02:58:23.225 INFO:tasks.cephadm:Deploying osd.1 on smithi033 with /dev/vg_nvme/lv_3... 2024-04-05T02:58:23.225 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-05T02:58:23.887 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:58:24.627 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[29244]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T02:58:24.627 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[29244]: osdmap e6: 1 total, 0 up, 1 in 2024-04-05T02:58:24.627 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:24.627 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[29244]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:58:24.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[40008]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T02:58:24.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[40008]: osdmap e6: 1 total, 0 up, 1 in 2024-04-05T02:58:24.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:24.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-mon[40008]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:58:24.628 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[43943]: 2024-04-05T02:58:24.320+0000 7f18e1b3c700 -1 osd.0 0 waiting for initial osdmap 2024-04-05T02:58:24.628 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:58:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[43943]: 2024-04-05T02:58:24.327+0000 7f18dc937700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:58:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:24 smithi151 ceph-mon[32789]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T02:58:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:24 smithi151 ceph-mon[32789]: osdmap e6: 1 total, 0 up, 1 in 2024-04-05T02:58:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:24 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:24.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:24 smithi151 ceph-mon[32789]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:58:25.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[29244]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:25.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[29244]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:58:25.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[29244]: osdmap e7: 1 total, 0 up, 1 in 2024-04-05T02:58:25.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.449 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[40008]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:25.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[40008]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:58:25.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[40008]: osdmap e7: 1 total, 0 up, 1 in 2024-04-05T02:58:25.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:25 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.713 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-05T02:58:25.713 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-05T02:58:25.713 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-04-05T02:58:25.713 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-04-05T02:58:25.713 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0136078 s, 771 MB/s 2024-04-05T02:58:25.713 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-04-05T02:58:25.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:25 smithi151 ceph-mon[32789]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:25.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:25 smithi151 ceph-mon[32789]: from='osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:58:25.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:25 smithi151 ceph-mon[32789]: osdmap e7: 1 total, 0 up, 1 in 2024-04-05T02:58:25.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:25 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:25 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:25.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:25 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:26.420 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi033:vg_nvme/lv_3 2024-04-05T02:58:26.608 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T02:58:26.608 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[29244]: osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893] boot 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[29244]: osdmap e8: 1 total, 1 up, 1 in 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[40008]: osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893] boot 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[40008]: osdmap e8: 1 total, 1 up, 1 in 2024-04-05T02:58:26.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:26 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:26 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T02:58:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:26 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T02:58:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:26 smithi151 ceph-mon[32789]: osd.0 [v2:172.21.15.33:6802/122630893,v1:172.21.15.33:6803/122630893] boot 2024-04-05T02:58:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:26 smithi151 ceph-mon[32789]: osdmap e8: 1 total, 1 up, 1 in 2024-04-05T02:58:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:26 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:58:26.882 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:58:27.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:27 smithi033 ceph-mon[29244]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:27.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:27 smithi033 ceph-mon[29244]: osdmap e9: 1 total, 1 up, 1 in 2024-04-05T02:58:27.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:27 smithi033 ceph-mon[40008]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:27.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:27 smithi033 ceph-mon[40008]: osdmap e9: 1 total, 1 up, 1 in 2024-04-05T02:58:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:27 smithi151 ceph-mon[32789]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:58:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:27 smithi151 ceph-mon[32789]: osdmap e9: 1 total, 1 up, 1 in 2024-04-05T02:58:29.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi033 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi033 to 16069M 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[29244]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi033 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi033 to 16069M 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:29 smithi033 ceph-mon[40008]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi033 2024-04-05T02:58:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi033 to 16069M 2024-04-05T02:58:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:29.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:29.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:29.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:29 smithi151 ceph-mon[32789]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:30.477 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[29244]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:30.478 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:30.478 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:30.478 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:30.478 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[40008]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:30.478 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:30.478 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:30.478 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:30 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:30 smithi151 ceph-mon[32789]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:30 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:30 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:30 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:31.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:31 smithi033 ceph-mon[29244]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:31.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:31 smithi033 ceph-mon[40008]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:31 smithi151 ceph-mon[32789]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3124317791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "046e9421-82b9-40de-b66d-958962a69c97"}]: dispatch 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3124317791' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "046e9421-82b9-40de-b66d-958962a69c97"}]': finished 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[29244]: osdmap e10: 2 total, 1 up, 2 in 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3124317791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "046e9421-82b9-40de-b66d-958962a69c97"}]: dispatch 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3124317791' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "046e9421-82b9-40de-b66d-958962a69c97"}]': finished 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[40008]: osdmap e10: 2 total, 1 up, 2 in 2024-04-05T02:58:32.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:32 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:32 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3124317791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "046e9421-82b9-40de-b66d-958962a69c97"}]: dispatch 2024-04-05T02:58:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:32 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3124317791' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "046e9421-82b9-40de-b66d-958962a69c97"}]': finished 2024-04-05T02:58:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:32 smithi151 ceph-mon[32789]: osdmap e10: 2 total, 1 up, 2 in 2024-04-05T02:58:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:32 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:33.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:33 smithi033 ceph-mon[40008]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:33.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:33 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1976433858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:33.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:33 smithi033 ceph-mon[29244]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:33.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:33 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1976433858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:33.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:33 smithi151 ceph-mon[32789]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:33.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:33 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1976433858' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:35.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:35 smithi033 ceph-mon[29244]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:35.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:35 smithi033 ceph-mon[40008]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:35 smithi151 ceph-mon[32789]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:37.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:37 smithi033 ceph-mon[29244]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:37.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:37 smithi033 ceph-mon[40008]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:37 smithi151 ceph-mon[32789]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:38.561 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:58:38.561 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:38.561 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:58:38.561 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:38.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:58:38.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:39.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:39 smithi033 ceph-mon[29244]: Deploying daemon osd.1 on smithi033 2024-04-05T02:58:39.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:39 smithi033 ceph-mon[29244]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:39.398 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:39 smithi033 ceph-mon[40008]: Deploying daemon osd.1 on smithi033 2024-04-05T02:58:39.398 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:39 smithi033 ceph-mon[40008]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:39.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:39 smithi151 ceph-mon[32789]: Deploying daemon osd.1 on smithi033 2024-04-05T02:58:39.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:39 smithi151 ceph-mon[32789]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:41 smithi151 ceph-mon[32789]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:41.603 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:41 smithi033 ceph-mon[29244]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:41.603 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:41 smithi033 ceph-mon[40008]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:43.601 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[29244]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:43.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:43.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:43.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:43.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[40008]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:43.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:43.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:43.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:43 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:43 smithi151 ceph-mon[32789]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:43 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:43 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:43 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:45.540 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:45 smithi033 ceph-mon[29244]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:45.540 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:45 smithi033 ceph-mon[40008]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:45 smithi151 ceph-mon[32789]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:46.218 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 1 on host 'smithi033' 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:58:46.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:46.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:46.914 DEBUG:teuthology.orchestra.run.smithi033:osd.1> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.1.service 2024-04-05T02:58:46.916 INFO:tasks.cephadm:Deploying osd.2 on smithi033 with /dev/vg_nvme/lv_2... 2024-04-05T02:58:46.916 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-05T02:58:47.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[29244]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:47.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[29244]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[29244]: osdmap e11: 2 total, 1 up, 2 in 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[29244]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[40008]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[40008]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[40008]: osdmap e11: 2 total, 1 up, 2 in 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:47.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-mon[40008]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:58:47.482 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[49782]: 2024-04-05T02:58:47.405+0000 7f273fb06700 -1 osd.1 0 waiting for initial osdmap 2024-04-05T02:58:47.482 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:58:47 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[49782]: 2024-04-05T02:58:47.409+0000 7f273c905700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:58:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:47 smithi151 ceph-mon[32789]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:47 smithi151 ceph-mon[32789]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T02:58:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:47 smithi151 ceph-mon[32789]: osdmap e11: 2 total, 1 up, 2 in 2024-04-05T02:58:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:47 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:47 smithi151 ceph-mon[32789]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:58:47.982 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:58:48.538 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[40008]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:58:48.538 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[40008]: osdmap e12: 2 total, 1 up, 2 in 2024-04-05T02:58:48.538 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:48.538 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:48.539 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[29244]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:58:48.539 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[29244]: osdmap e12: 2 total, 1 up, 2 in 2024-04-05T02:58:48.539 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:48.539 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:48 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:48 smithi151 ceph-mon[32789]: from='osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:58:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:48 smithi151 ceph-mon[32789]: osdmap e12: 2 total, 1 up, 2 in 2024-04-05T02:58:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:48 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:48 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[40008]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[40008]: osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148] boot 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[40008]: osdmap e13: 2 total, 2 up, 2 in 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T02:58:49.435 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[29244]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:49.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[29244]: osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148] boot 2024-04-05T02:58:49.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[29244]: osdmap e13: 2 total, 2 up, 2 in 2024-04-05T02:58:49.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:49 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:49 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T02:58:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:49 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T02:58:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:49 smithi151 ceph-mon[32789]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:58:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:49 smithi151 ceph-mon[32789]: osd.1 [v2:172.21.15.33:6810/2920348148,v1:172.21.15.33:6811/2920348148] boot 2024-04-05T02:58:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:49 smithi151 ceph-mon[32789]: osdmap e13: 2 total, 2 up, 2 in 2024-04-05T02:58:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:49 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:58:50.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: osdmap e14: 2 total, 2 up, 2 in 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: osdmap e14: 2 total, 2 up, 2 in 2024-04-05T02:58:50.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:50.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:50.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:50.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:50.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: osdmap e14: 2 total, 2 up, 2 in 2024-04-05T02:58:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:58:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:58:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:58:50.849 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-05T02:58:50.850 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-05T02:58:50.850 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-04-05T02:58:50.850 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-04-05T02:58:50.850 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110519 s, 949 MB/s 2024-04-05T02:58:50.850 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-04-05T02:58:51.466 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi033:vg_nvme/lv_2 2024-04-05T02:58:51.727 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:58:51.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:51 smithi033 ceph-mon[40008]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:51.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:51 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi033 2024-04-05T02:58:51.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:51 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi033 to 8034M 2024-04-05T02:58:51.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:51 smithi033 ceph-mon[29244]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:51.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:51 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi033 2024-04-05T02:58:51.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:51 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi033 to 8034M 2024-04-05T02:58:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:51 smithi151 ceph-mon[32789]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:51 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi033 2024-04-05T02:58:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:51 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi033 to 8034M 2024-04-05T02:58:53.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:53 smithi033 ceph-mon[29244]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:53 smithi033 ceph-mon[40008]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:53.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:53 smithi151 ceph-mon[32789]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:54.587 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[40008]: from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:54.587 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:54.587 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:54.587 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:54.587 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[29244]: from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:54.587 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:54.587 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:54.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:54 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:54 smithi151 ceph-mon[32789]: from='client.24176 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:58:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:54 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:58:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:54 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:58:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:54 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:58:55.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:55 smithi033 ceph-mon[29244]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:55.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:55 smithi033 ceph-mon[40008]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:55.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:55 smithi151 ceph-mon[32789]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3201863095' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]: dispatch 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]: dispatch 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]': finished 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[29244]: osdmap e15: 3 total, 2 up, 3 in 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3201863095' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]: dispatch 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]: dispatch 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]': finished 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[40008]: osdmap e15: 3 total, 2 up, 3 in 2024-04-05T02:58:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:58:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:56 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3201863095' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]: dispatch 2024-04-05T02:58:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:56 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]: dispatch 2024-04-05T02:58:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:56 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc74077e-8e35-4d06-ba7d-ad0b05e4e515"}]': finished 2024-04-05T02:58:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:56 smithi151 ceph-mon[32789]: osdmap e15: 3 total, 2 up, 3 in 2024-04-05T02:58:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:58:57.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:57 smithi033 ceph-mon[29244]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:57.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:57 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2590776767' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:57 smithi033 ceph-mon[40008]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:57 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2590776767' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:57.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:57 smithi151 ceph-mon[32789]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:57.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:57 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2590776767' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:58:59.454 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:58:59 smithi033 ceph-mon[29244]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:59.458 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:58:59 smithi033 ceph-mon[40008]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:58:59.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:58:59 smithi151 ceph-mon[32789]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:01.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:01 smithi033 ceph-mon[29244]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:01.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:01 smithi033 ceph-mon[40008]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:01.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:01 smithi151 ceph-mon[32789]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:02.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:59:02.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:02 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:02.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:59:02.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:02 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:02.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:59:02.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:02 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:03.656 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:03 smithi033 ceph-mon[29244]: Deploying daemon osd.2 on smithi033 2024-04-05T02:59:03.656 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:03 smithi033 ceph-mon[29244]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:03.656 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:03 smithi033 ceph-mon[40008]: Deploying daemon osd.2 on smithi033 2024-04-05T02:59:03.656 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:03 smithi033 ceph-mon[40008]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:03.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:03 smithi151 ceph-mon[32789]: Deploying daemon osd.2 on smithi033 2024-04-05T02:59:03.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:03 smithi151 ceph-mon[32789]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:05.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:05 smithi033 ceph-mon[29244]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:05.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:05 smithi033 ceph-mon[40008]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:05.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:05 smithi151 ceph-mon[32789]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:07.719 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[29244]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:07.719 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:07.719 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:07.719 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:07.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[40008]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:07.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:07.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:07.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:07 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:07.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:07 smithi151 ceph-mon[32789]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:07.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:07 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:07.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:07 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:07.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:07 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:09.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:09 smithi033 ceph-mon[29244]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:09 smithi033 ceph-mon[40008]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:09.763 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 2 on host 'smithi033' 2024-04-05T02:59:09.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:09 smithi151 ceph-mon[32789]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:10.626 DEBUG:teuthology.orchestra.run.smithi033:osd.2> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.2.service 2024-04-05T02:59:10.629 INFO:tasks.cephadm:Deploying osd.3 on smithi033 with /dev/vg_nvme/lv_1... 2024-04-05T02:59:10.629 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-05T02:59:10.957 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[29244]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:59:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[29244]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:10.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:10.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:10.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[40008]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:59:10.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:10 smithi033 ceph-mon[40008]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:11.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:10 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:11.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:10 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:11.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:10 smithi151 ceph-mon[32789]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:59:11.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:10 smithi151 ceph-mon[32789]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[40008]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[40008]: osdmap e16: 3 total, 2 up, 3 in 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[40008]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[29244]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T02:59:11.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[29244]: osdmap e16: 3 total, 2 up, 3 in 2024-04-05T02:59:11.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:11.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[29244]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:11.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:11.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:11.985 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[55581]: 2024-04-05T02:59:11.724+0000 7f4b66cbc700 -1 osd.2 0 waiting for initial osdmap 2024-04-05T02:59:11.985 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[55581]: 2024-04-05T02:59:11.729+0000 7f4b63abb700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:59:12.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:11 smithi151 ceph-mon[32789]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T02:59:12.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:11 smithi151 ceph-mon[32789]: osdmap e16: 3 total, 2 up, 3 in 2024-04-05T02:59:12.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:11 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:12.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:11 smithi151 ceph-mon[32789]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:12.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:11 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:12.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:11 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:12.839 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-05T02:59:12.839 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-05T02:59:12.839 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-04-05T02:59:12.839 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-04-05T02:59:12.839 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121436 s, 863 MB/s 2024-04-05T02:59:12.840 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-04-05T02:59:12.840 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[29244]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:59:12.840 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[29244]: osdmap e17: 3 total, 2 up, 3 in 2024-04-05T02:59:12.840 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:12.840 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:12.840 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[29244]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:12.840 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[40008]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:59:12.841 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[40008]: osdmap e17: 3 total, 2 up, 3 in 2024-04-05T02:59:12.841 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:12.841 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:12.841 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:12 smithi033 ceph-mon[40008]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:12 smithi151 ceph-mon[32789]: from='osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:59:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:12 smithi151 ceph-mon[32789]: osdmap e17: 3 total, 2 up, 3 in 2024-04-05T02:59:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:12 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:12 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:12 smithi151 ceph-mon[32789]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:59:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T02:59:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T02:59:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[40008]: osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518] boot 2024-04-05T02:59:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[40008]: osdmap e18: 3 total, 3 up, 3 in 2024-04-05T02:59:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:13.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T02:59:13.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T02:59:13.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:13.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[29244]: osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518] boot 2024-04-05T02:59:13.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[29244]: osdmap e18: 3 total, 3 up, 3 in 2024-04-05T02:59:13.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:14.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:13 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T02:59:14.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:13 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T02:59:14.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:14.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:13 smithi151 ceph-mon[32789]: osd.2 [v2:172.21.15.33:6818/3206964518,v1:172.21.15.33:6819/3206964518] boot 2024-04-05T02:59:14.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:13 smithi151 ceph-mon[32789]: osdmap e18: 3 total, 3 up, 3 in 2024-04-05T02:59:14.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:59:14.126 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi033:vg_nvme/lv_1 2024-04-05T02:59:14.660 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:14 smithi151 ceph-mon[32789]: osdmap e19: 3 total, 3 up, 3 in 2024-04-05T02:59:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:14 smithi151 ceph-mon[32789]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:15.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:14 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-05T02:59:15.044 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:14 smithi033 ceph-mon[29244]: osdmap e19: 3 total, 3 up, 3 in 2024-04-05T02:59:15.044 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:14 smithi033 ceph-mon[29244]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:15.045 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:14 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-05T02:59:15.045 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:14 smithi033 ceph-mon[40008]: osdmap e19: 3 total, 3 up, 3 in 2024-04-05T02:59:15.045 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:14 smithi033 ceph-mon[40008]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:15.045 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:14 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-05T02:59:16.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:15 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-05T02:59:16.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:15 smithi151 ceph-mon[32789]: osdmap e20: 3 total, 3 up, 3 in 2024-04-05T02:59:16.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:15 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:16.162 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-05T02:59:16.162 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:15 smithi033 ceph-mon[29244]: osdmap e20: 3 total, 3 up, 3 in 2024-04-05T02:59:16.162 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:15 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:16.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:15 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-05T02:59:16.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:15 smithi033 ceph-mon[40008]: osdmap e20: 3 total, 3 up, 3 in 2024-04-05T02:59:16.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:15 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:16.730 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59780]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:59:16.731 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59780]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:16.731 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59780]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:16.731 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59780]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:17.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:59:17.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: osdmap e21: 3 total, 3 up, 3 in 2024-04-05T02:59:17.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi033 2024-04-05T02:59:17.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi033 to 5356M 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:16 smithi151 ceph-mon[32789]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:17.205 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59836]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:59:17.206 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59836]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:17.206 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59836]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:17.206 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 02:59:16 smithi033 sudo[59836]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:17.206 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:59:17.206 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: osdmap e21: 3 total, 3 up, 3 in 2024-04-05T02:59:17.206 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi033 2024-04-05T02:59:17.206 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.206 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi033 to 5356M 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[29244]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: osdmap e21: 3 total, 3 up, 3 in 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi033 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi033 to 5356M 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:17.208 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.209 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:17.209 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:17.209 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:17.209 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:17.209 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:16 smithi033 ceph-mon[40008]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:17.469 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59921]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:59:17.469 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59921]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:17.469 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59921]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:17.469 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59921]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:17.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59933]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:59:17.731 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59933]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:17.731 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59933]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:17.731 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59933]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:18.014 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[40008]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:18.015 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:59:18.015 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:59:18.015 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:18.015 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:18.015 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[40008]: osdmap e22: 3 total, 3 up, 3 in 2024-04-05T02:59:18.015 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59957]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:59:18.015 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59957]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:18.015 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59957]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:18.016 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:17 smithi033 sudo[59957]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:18.016 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[29244]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:18.016 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:59:18.016 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:59:18.016 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:18.016 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:18.016 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:17 smithi033 ceph-mon[29244]: osdmap e22: 3 total, 3 up, 3 in 2024-04-05T02:59:18.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:17 smithi151 ceph-mon[32789]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:18.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:17 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:59:18.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:17 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:59:18.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:17 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:18.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:17 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:18.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:17 smithi151 ceph-mon[32789]: osdmap e22: 3 total, 3 up, 3 in 2024-04-05T02:59:18.275 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59965]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:59:18.275 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59965]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:18.275 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59965]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:18.275 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59965]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:18.544 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59988]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:59:18.545 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59988]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:18.545 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59988]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:18.545 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59988]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:18.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59998]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:59:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59998]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59998]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:18 smithi033 sudo[59998]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:19.040 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:18 smithi151 sudo[35371]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:59:19.040 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:18 smithi151 sudo[35371]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:19.041 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:18 smithi151 sudo[35371]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:19.041 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:18 smithi151 sudo[35371]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:19.325 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:19.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 sudo[35374]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:59:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 sudo[35374]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 sudo[35374]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 sudo[35374]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:19.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:19.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:19.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:19.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:19.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:19.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:19.589 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60047]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:59:19.589 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60047]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:19.589 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60047]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:19.589 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60047]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:19.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60050]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:59:19.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60050]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:59:19.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60050]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:59:19.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:19 smithi033 sudo[60050]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:59:20.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:20.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2075280396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95b643ed-ea3a-4e88-a2bf-6cc4f398df73"}]: dispatch 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2075280396' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95b643ed-ea3a-4e88-a2bf-6cc4f398df73"}]': finished 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: osdmap e23: 4 total, 3 up, 4 in 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:20.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2075280396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95b643ed-ea3a-4e88-a2bf-6cc4f398df73"}]: dispatch 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2075280396' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95b643ed-ea3a-4e88-a2bf-6cc4f398df73"}]': finished 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: osdmap e23: 4 total, 3 up, 4 in 2024-04-05T02:59:20.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-05T02:59:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:59:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:20.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:20.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:20.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2075280396' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95b643ed-ea3a-4e88-a2bf-6cc4f398df73"}]: dispatch 2024-04-05T02:59:20.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:59:20.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2075280396' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95b643ed-ea3a-4e88-a2bf-6cc4f398df73"}]': finished 2024-04-05T02:59:20.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: osdmap e23: 4 total, 3 up, 4 in 2024-04-05T02:59:20.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:21.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:21 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1287906142' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:59:21.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:21 smithi033 ceph-mon[29244]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:21.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:21 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1287906142' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:59:21.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:21 smithi033 ceph-mon[40008]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:21.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:21 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1287906142' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:59:21.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:21 smithi151 ceph-mon[32789]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:23.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:23 smithi033 ceph-mon[40008]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:23.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:23 smithi033 ceph-mon[29244]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:23 smithi151 ceph-mon[32789]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:25.473 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:25 smithi033 ceph-mon[40008]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:25.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:25 smithi033 ceph-mon[29244]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:25.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:25 smithi151 ceph-mon[32789]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:26.452 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:26 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:59:26.452 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:26 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:26.452 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:26 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:59:26.452 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:26 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:26 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:59:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:26 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:27.542 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:27 smithi033 ceph-mon[29244]: Deploying daemon osd.3 on smithi033 2024-04-05T02:59:27.542 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:27 smithi033 ceph-mon[29244]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.4 KiB/s rd, 39 KiB/s wr, 3 op/s 2024-04-05T02:59:27.542 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:27 smithi033 ceph-mon[40008]: Deploying daemon osd.3 on smithi033 2024-04-05T02:59:27.542 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:27 smithi033 ceph-mon[40008]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.4 KiB/s rd, 39 KiB/s wr, 3 op/s 2024-04-05T02:59:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:27 smithi151 ceph-mon[32789]: Deploying daemon osd.3 on smithi033 2024-04-05T02:59:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:27 smithi151 ceph-mon[32789]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.4 KiB/s rd, 39 KiB/s wr, 3 op/s 2024-04-05T02:59:29.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:29 smithi033 ceph-mon[29244]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.0 KiB/s rd, 30 KiB/s wr, 2 op/s 2024-04-05T02:59:29.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:29 smithi033 ceph-mon[40008]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.0 KiB/s rd, 30 KiB/s wr, 2 op/s 2024-04-05T02:59:29.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:29 smithi151 ceph-mon[32789]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.0 KiB/s rd, 30 KiB/s wr, 2 op/s 2024-04-05T02:59:31.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[40008]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 864 B/s rd, 25 KiB/s wr, 2 op/s 2024-04-05T02:59:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[29244]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 864 B/s rd, 25 KiB/s wr, 2 op/s 2024-04-05T02:59:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:31.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:31 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:31 smithi151 ceph-mon[32789]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 864 B/s rd, 25 KiB/s wr, 2 op/s 2024-04-05T02:59:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:31 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:31 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:31 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:33.630 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:33 smithi033 ceph-mon[29244]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-05T02:59:33.630 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:33 smithi033 ceph-mon[40008]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-05T02:59:33.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:33 smithi151 ceph-mon[32789]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-05T02:59:34.280 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 3 on host 'smithi033' 2024-04-05T02:59:34.855 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[40008]: from='osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[40008]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[29244]: from='osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:59:34.856 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[29244]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:59:34.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:34 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.881 DEBUG:teuthology.orchestra.run.smithi033:osd.3> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.3.service 2024-04-05T02:59:34.883 INFO:tasks.cephadm:Deploying osd.4 on smithi151 with /dev/vg_nvme/lv_4... 2024-04-05T02:59:34.884 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-05T02:59:34.914 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:34 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.914 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:34 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.914 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:34 smithi151 ceph-mon[32789]: from='osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:59:34.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:34 smithi151 ceph-mon[32789]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:59:34.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:34 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:34.915 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:34 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:35.141 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[29244]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[29244]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[29244]: osdmap e24: 4 total, 3 up, 4 in 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[29244]: from='osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[29244]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[40008]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-05T02:59:35.766 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[40008]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T02:59:35.767 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[40008]: osdmap e24: 4 total, 3 up, 4 in 2024-04-05T02:59:35.767 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:35.767 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[40008]: from='osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:35.767 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-mon[40008]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:35.767 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[61729]: 2024-04-05T02:59:35.653+0000 7f8e4c821700 -1 osd.3 0 waiting for initial osdmap 2024-04-05T02:59:35.767 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 02:59:35 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[61729]: 2024-04-05T02:59:35.658+0000 7f8e4861e700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:59:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:35 smithi151 ceph-mon[32789]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-05T02:59:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:35 smithi151 ceph-mon[32789]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T02:59:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:35 smithi151 ceph-mon[32789]: osdmap e24: 4 total, 3 up, 4 in 2024-04-05T02:59:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:35 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:35 smithi151 ceph-mon[32789]: from='osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:35 smithi151 ceph-mon[32789]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T02:59:36.697 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[40008]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:59:36.697 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[40008]: osdmap e25: 4 total, 3 up, 4 in 2024-04-05T02:59:36.697 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:36.697 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:36.697 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[40008]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:36.698 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[29244]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:59:36.698 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[29244]: osdmap e25: 4 total, 3 up, 4 in 2024-04-05T02:59:36.698 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:36.698 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:36.698 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:36 smithi033 ceph-mon[29244]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:36 smithi151 ceph-mon[32789]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-04-05T02:59:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:36 smithi151 ceph-mon[32789]: osdmap e25: 4 total, 3 up, 4 in 2024-04-05T02:59:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:36 smithi151 ceph-mon[32789]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:59:36.976 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-05T02:59:36.977 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-05T02:59:36.977 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-05T02:59:36.977 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-05T02:59:36.977 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012321 s, 851 MB/s 2024-04-05T02:59:36.977 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-05T02:59:37.576 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi151:vg_nvme/lv_4 2024-04-05T02:59:37.862 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T02:59:37.862 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T02:59:37.862 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[40008]: osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270] boot 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[40008]: osdmap e26: 4 total, 4 up, 4 in 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[29244]: osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270] boot 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[29244]: osdmap e26: 4 total, 4 up, 4 in 2024-04-05T02:59:37.863 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:37 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:37.878 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:59:38.002 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:37 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T02:59:38.002 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:37 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T02:59:38.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:37 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:38.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:37 smithi151 ceph-mon[32789]: osd.3 [v2:172.21.15.33:6826/1608936270,v1:172.21.15.33:6827/1608936270] boot 2024-04-05T02:59:38.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:37 smithi151 ceph-mon[32789]: osdmap e26: 4 total, 4 up, 4 in 2024-04-05T02:59:38.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:37 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:59:38.925 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: osdmap e27: 4 total, 4 up, 4 in 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi033 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi033 to 4017M 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:38.927 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:38.927 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.927 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:38.927 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:38 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: osdmap e27: 4 total, 4 up, 4 in 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi033 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi033 to 4017M 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: osdmap e27: 4 total, 4 up, 4 in 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi033 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi033 to 4017M 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:38.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:38 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:40.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:40 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:59:40.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:40 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:59:40.331 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:40 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:40.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:40 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:59:40.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:40 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:59:40.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:40 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:40 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:59:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:40 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:59:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:40 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:41.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:41 smithi151 ceph-mon[32789]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:41.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:41 smithi151 ceph-mon[32789]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:41.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:41 smithi033 ceph-mon[29244]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:41.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:41 smithi033 ceph-mon[29244]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:41.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:41 smithi033 ceph-mon[40008]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:41.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:41 smithi033 ceph-mon[40008]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:42.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:42 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/210660232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]: dispatch 2024-04-05T02:59:42.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:42 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]: dispatch 2024-04-05T02:59:42.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:42 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]': finished 2024-04-05T02:59:42.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:42 smithi151 ceph-mon[32789]: osdmap e28: 5 total, 4 up, 5 in 2024-04-05T02:59:42.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:42 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/210660232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]: dispatch 2024-04-05T02:59:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]: dispatch 2024-04-05T02:59:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]': finished 2024-04-05T02:59:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[29244]: osdmap e28: 5 total, 4 up, 5 in 2024-04-05T02:59:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/210660232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]: dispatch 2024-04-05T02:59:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]: dispatch 2024-04-05T02:59:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70bbcc30-2695-4251-927c-63b0251eca78"}]': finished 2024-04-05T02:59:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[40008]: osdmap e28: 5 total, 4 up, 5 in 2024-04-05T02:59:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:42 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:43.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:43 smithi151 ceph-mon[32789]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:43.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/4122287282' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:59:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:43 smithi033 ceph-mon[29244]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/4122287282' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:59:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:43 smithi033 ceph-mon[40008]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:43 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/4122287282' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:59:45.442 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:45 smithi151 ceph-mon[32789]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:45.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:45 smithi033 ceph-mon[29244]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:45.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:45 smithi033 ceph-mon[40008]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:47.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:47 smithi151 ceph-mon[32789]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:47 smithi033 ceph-mon[29244]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:47 smithi033 ceph-mon[40008]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:48 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:59:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:48 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:48.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:48 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:59:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:48 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:48.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:48 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:59:48.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:48 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:49.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:49 smithi151 ceph-mon[32789]: Deploying daemon osd.4 on smithi151 2024-04-05T02:59:49.454 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:49 smithi151 ceph-mon[32789]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:49.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:49 smithi033 ceph-mon[29244]: Deploying daemon osd.4 on smithi151 2024-04-05T02:59:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:49 smithi033 ceph-mon[29244]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:49 smithi033 ceph-mon[40008]: Deploying daemon osd.4 on smithi151 2024-04-05T02:59:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:49 smithi033 ceph-mon[40008]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:51 smithi151 ceph-mon[32789]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:51 smithi033 ceph-mon[29244]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:51.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:51 smithi033 ceph-mon[40008]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:53.384 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:53 smithi151 ceph-mon[32789]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:53.385 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:53 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:53.385 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:53 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:53.385 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:53 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:53.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[29244]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:53.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:53.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:53.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[40008]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:53 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:55.523 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:55 smithi151 ceph-mon[32789]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:55.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:55 smithi033 ceph-mon[29244]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:55.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:55 smithi033 ceph-mon[40008]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:55.802 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 4 on host 'smithi151' 2024-04-05T02:59:56.469 DEBUG:teuthology.orchestra.run.smithi151:osd.4> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.4.service 2024-04-05T02:59:56.471 INFO:tasks.cephadm:Deploying osd.5 on smithi151 with /dev/vg_nvme/lv_3... 2024-04-05T02:59:56.471 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-05T02:59:56.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.720 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.720 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.720 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:56 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.720 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:56 smithi151 ceph-mon[32789]: from='osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:59:56.720 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:56 smithi151 ceph-mon[32789]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:59:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[29244]: from='osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[29244]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[40008]: from='osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:59:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:56 smithi033 ceph-mon[40008]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:59:57.483 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-mon[32789]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:57.483 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-mon[32789]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-05T02:59:57.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-mon[32789]: osdmap e29: 5 total, 4 up, 5 in 2024-04-05T02:59:57.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:57.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-mon[32789]: from='osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T02:59:57.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-mon[32789]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T02:59:57.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[29244]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[29244]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[29244]: osdmap e29: 5 total, 4 up, 5 in 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[29244]: from='osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[29244]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[40008]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[40008]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[40008]: osdmap e29: 5 total, 4 up, 5 in 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[40008]: from='osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T02:59:57.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:57 smithi033 ceph-mon[40008]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T02:59:58.004 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:59:58.028 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T02:59:57.792+0000 7f2189315700 -1 osd.4 0 waiting for initial osdmap 2024-04-05T02:59:58.028 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 02:59:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T02:59:57.797+0000 7f218390f700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:59:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:58 smithi151 ceph-mon[32789]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T02:59:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:58 smithi151 ceph-mon[32789]: osdmap e30: 5 total, 4 up, 5 in 2024-04-05T02:59:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:58 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:58 smithi151 ceph-mon[32789]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:59.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[29244]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[29244]: osdmap e30: 5 total, 4 up, 5 in 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[29244]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[40008]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[40008]: osdmap e30: 5 total, 4 up, 5 in 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:59:59.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:58 smithi033 ceph-mon[40008]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T03:00:00.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T03:00:00.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T03:00:00.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039] boot 2024-04-05T03:00:00.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: osdmap e31: 5 total, 5 up, 5 in 2024-04-05T03:00:00.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:00:00.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: osdmap e32: 5 total, 5 up, 5 in 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi151 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi151 to 17088M 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:00.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 02:59:59 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039] boot 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: osdmap e31: 5 total, 5 up, 5 in 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: osdmap e32: 5 total, 5 up, 5 in 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi151 2024-04-05T03:00:00.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi151 to 17088M 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: osd.4 [v2:172.21.15.151:6800/1041236039,v1:172.21.15.151:6801/1041236039] boot 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: osdmap e31: 5 total, 5 up, 5 in 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: osdmap e32: 5 total, 5 up, 5 in 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi151 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi151 to 17088M 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:00.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.234 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:00.234 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:00.234 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:00.234 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 02:59:59 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:00.577 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-05T03:00:00.577 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-05T03:00:00.577 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-05T03:00:00.577 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-05T03:00:00.577 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124891 s, 840 MB/s 2024-04-05T03:00:00.577 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-05T03:00:00.904 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:00 smithi151 ceph-mon[32789]: osdmap e33: 5 total, 5 up, 5 in 2024-04-05T03:00:00.904 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:00 smithi151 ceph-mon[32789]: overall HEALTH_OK 2024-04-05T03:00:00.904 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:00 smithi151 ceph-mon[32789]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:01.222 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi151:vg_nvme/lv_3 2024-04-05T03:00:01.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:00 smithi033 ceph-mon[29244]: osdmap e33: 5 total, 5 up, 5 in 2024-04-05T03:00:01.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:00 smithi033 ceph-mon[29244]: overall HEALTH_OK 2024-04-05T03:00:01.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:00 smithi033 ceph-mon[29244]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:01.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:00 smithi033 ceph-mon[40008]: osdmap e33: 5 total, 5 up, 5 in 2024-04-05T03:00:01.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:00 smithi033 ceph-mon[40008]: overall HEALTH_OK 2024-04-05T03:00:01.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:00 smithi033 ceph-mon[40008]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:01.474 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:00:01.917 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:01 smithi151 ceph-mon[32789]: osdmap e34: 5 total, 5 up, 5 in 2024-04-05T03:00:02.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:01 smithi033 ceph-mon[29244]: osdmap e34: 5 total, 5 up, 5 in 2024-04-05T03:00:02.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:01 smithi033 ceph-mon[40008]: osdmap e34: 5 total, 5 up, 5 in 2024-04-05T03:00:02.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:02 smithi151 ceph-mon[32789]: osdmap e35: 5 total, 5 up, 5 in 2024-04-05T03:00:02.865 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:02 smithi151 ceph-mon[32789]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:03.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:02 smithi033 ceph-mon[29244]: osdmap e35: 5 total, 5 up, 5 in 2024-04-05T03:00:03.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:02 smithi033 ceph-mon[29244]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:03.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:02 smithi033 ceph-mon[40008]: osdmap e35: 5 total, 5 up, 5 in 2024-04-05T03:00:03.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:02 smithi033 ceph-mon[40008]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:03.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:03 smithi151 ceph-mon[32789]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:03.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:03 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:03.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:03 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:03.926 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:03 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:04.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[40008]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:04.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:04.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:04.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:04.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[29244]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:04.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:04.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:04.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:03 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:04.890 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:04 smithi151 ceph-mon[32789]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:05.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:04 smithi033 ceph-mon[29244]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:05.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:04 smithi033 ceph-mon[40008]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:06.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/2349684558' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]: dispatch 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]: dispatch 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]': finished 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[40008]: osdmap e36: 6 total, 5 up, 6 in 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/2349684558' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]: dispatch 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]: dispatch 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]': finished 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[29244]: osdmap e36: 6 total, 5 up, 6 in 2024-04-05T03:00:06.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:05 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:05 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/2349684558' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]: dispatch 2024-04-05T03:00:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:05 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]: dispatch 2024-04-05T03:00:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:05 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7dc85ee-9174-4774-83fc-249a6f3b29d9"}]': finished 2024-04-05T03:00:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:05 smithi151 ceph-mon[32789]: osdmap e36: 6 total, 5 up, 6 in 2024-04-05T03:00:06.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:05 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:07.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:06 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/1784589096' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:07.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:06 smithi033 ceph-mon[40008]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:00:07.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:06 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/1784589096' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:07.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:06 smithi033 ceph-mon[29244]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:00:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:06 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/1784589096' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:06 smithi151 ceph-mon[32789]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:00:09.505 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:09 smithi151 ceph-mon[32789]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-05T03:00:09.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:09 smithi033 ceph-mon[29244]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-05T03:00:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:09 smithi033 ceph-mon[40008]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-05T03:00:11.644 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:11 smithi151 ceph-mon[32789]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-05T03:00:11.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:11 smithi033 ceph-mon[29244]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-05T03:00:11.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:11 smithi033 ceph-mon[40008]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-05T03:00:12.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:12 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:00:12.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:12 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:12.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:12 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:00:12.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:12 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:12.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:12 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:00:12.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:12 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:13.583 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:13 smithi151 ceph-mon[32789]: Deploying daemon osd.5 on smithi151 2024-04-05T03:00:13.583 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:13 smithi151 ceph-mon[32789]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:00:13.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:13 smithi033 ceph-mon[29244]: Deploying daemon osd.5 on smithi151 2024-04-05T03:00:13.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:13 smithi033 ceph-mon[29244]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:00:13.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:13 smithi033 ceph-mon[40008]: Deploying daemon osd.5 on smithi151 2024-04-05T03:00:13.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:13 smithi033 ceph-mon[40008]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:00:15.427 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:15 smithi151 ceph-mon[32789]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:00:15.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:15 smithi033 ceph-mon[29244]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:00:15.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:15 smithi033 ceph-mon[40008]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:00:17.637 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:17 smithi151 ceph-mon[32789]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:17.637 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:17 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:17.637 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:17 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:17.637 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:17 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:17.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[29244]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:17.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[40008]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:17 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:19 smithi151 ceph-mon[32789]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:19.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:19 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[29244]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[40008]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:19.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:19.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:19.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:19 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:20.327 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 5 on host 'smithi151' 2024-04-05T03:00:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:20 smithi151 ceph-mon[32789]: from='osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T03:00:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:20 smithi151 ceph-mon[32789]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T03:00:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:20 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:20.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[29244]: from='osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[29244]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[40008]: from='osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[40008]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:20.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:20 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:20.896 DEBUG:teuthology.orchestra.run.smithi151:osd.5> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.5.service 2024-04-05T03:00:20.898 INFO:tasks.cephadm:Deploying osd.6 on smithi151 with /dev/vg_nvme/lv_2... 2024-04-05T03:00:20.899 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-05T03:00:21.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-mon[32789]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:21.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-mon[32789]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-05T03:00:21.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-mon[32789]: osdmap e37: 6 total, 5 up, 6 in 2024-04-05T03:00:21.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:21.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-mon[32789]: from='osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:21.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-mon[32789]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:21.529 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:00:21.360+0000 7fe57ed74700 -1 osd.5 0 waiting for initial osdmap 2024-04-05T03:00:21.529 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:00:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:00:21.365+0000 7fe57836c700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:00:21.566 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:00:21.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[29244]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:21.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[29244]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-05T03:00:21.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[29244]: osdmap e37: 6 total, 5 up, 6 in 2024-04-05T03:00:21.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:21.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[29244]: from='osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:21.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[29244]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:21.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[40008]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:21.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[40008]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-05T03:00:21.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[40008]: osdmap e37: 6 total, 5 up, 6 in 2024-04-05T03:00:21.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:21.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[40008]: from='osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:21.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:21 smithi033 ceph-mon[40008]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:22.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:22 smithi151 ceph-mon[32789]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:00:22.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:22 smithi151 ceph-mon[32789]: osdmap e38: 6 total, 5 up, 6 in 2024-04-05T03:00:22.666 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:22.667 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:22 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:22.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[29244]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:00:22.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[29244]: osdmap e38: 6 total, 5 up, 6 in 2024-04-05T03:00:22.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:22.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[40008]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:00:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[40008]: osdmap e38: 6 total, 5 up, 6 in 2024-04-05T03:00:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:22 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:23.421 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:23 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T03:00:23.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:23 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T03:00:23.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:23 smithi151 ceph-mon[32789]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:23.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:23 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:23.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:23 smithi151 ceph-mon[32789]: osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192] boot 2024-04-05T03:00:23.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:23 smithi151 ceph-mon[32789]: osdmap e39: 6 total, 6 up, 6 in 2024-04-05T03:00:23.422 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:23 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:23.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T03:00:23.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[29244]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[29244]: osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192] boot 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[29244]: osdmap e39: 6 total, 6 up, 6 in 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[40008]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[40008]: osd.5 [v2:172.21.15.151:6808/2180428192,v1:172.21.15.151:6809/2180428192] boot 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[40008]: osdmap e39: 6 total, 6 up, 6 in 2024-04-05T03:00:23.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:23 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:00:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[29244]: osdmap e40: 6 total, 6 up, 6 in 2024-04-05T03:00:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[40008]: osdmap e40: 6 total, 6 up, 6 in 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:24 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:24 smithi151 ceph-mon[32789]: osdmap e40: 6 total, 6 up, 6 in 2024-04-05T03:00:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:24 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:24 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:24.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:24 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:24.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:24 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:24.928 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-05T03:00:24.928 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-05T03:00:24.928 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-05T03:00:24.929 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-05T03:00:24.929 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113153 s, 927 MB/s 2024-04-05T03:00:24.929 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-05T03:00:25.591 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi151 2024-04-05T03:00:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi151 to 8544M 2024-04-05T03:00:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: osdmap e41: 6 total, 6 up, 6 in 2024-04-05T03:00:25.592 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:25 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:25.616 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi151:vg_nvme/lv_2 2024-04-05T03:00:25.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi151 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi151 to 8544M 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: osdmap e41: 6 total, 6 up, 6 in 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi151 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi151 to 8544M 2024-04-05T03:00:25.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:25.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:25.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:25.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: osdmap e41: 6 total, 6 up, 6 in 2024-04-05T03:00:25.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:25 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:25.868 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:00:26.664 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:26 smithi151 ceph-mon[32789]: osdmap e42: 6 total, 6 up, 6 in 2024-04-05T03:00:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:26 smithi033 ceph-mon[29244]: osdmap e42: 6 total, 6 up, 6 in 2024-04-05T03:00:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:26 smithi033 ceph-mon[40008]: osdmap e42: 6 total, 6 up, 6 in 2024-04-05T03:00:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:27 smithi151 ceph-mon[32789]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:27 smithi151 ceph-mon[32789]: osdmap e43: 6 total, 6 up, 6 in 2024-04-05T03:00:27.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:27 smithi033 ceph-mon[29244]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:27.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:27 smithi033 ceph-mon[29244]: osdmap e43: 6 total, 6 up, 6 in 2024-04-05T03:00:27.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:27 smithi033 ceph-mon[40008]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:27.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:27 smithi033 ceph-mon[40008]: osdmap e43: 6 total, 6 up, 6 in 2024-04-05T03:00:28.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:28 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:28.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:28 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:28.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:28 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:28.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:28 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:28.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:28 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:28.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:28 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:28.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:28 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:28.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:28 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:28.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:28 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:29 smithi151 ceph-mon[32789]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:29 smithi151 ceph-mon[32789]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:29.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:29 smithi033 ceph-mon[29244]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:29.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:29 smithi033 ceph-mon[29244]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:29.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:29 smithi033 ceph-mon[40008]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:29.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:29 smithi033 ceph-mon[40008]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/1180092888' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]: dispatch 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]: dispatch 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]': finished 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[29244]: osdmap e44: 7 total, 6 up, 7 in 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/1180092888' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]: dispatch 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]: dispatch 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]': finished 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[40008]: osdmap e44: 7 total, 6 up, 7 in 2024-04-05T03:00:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:30 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:30 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/1180092888' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]: dispatch 2024-04-05T03:00:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:30 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]: dispatch 2024-04-05T03:00:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:30 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b492b365-76da-4aae-a854-26d49a49b7b2"}]': finished 2024-04-05T03:00:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:30 smithi151 ceph-mon[32789]: osdmap e44: 7 total, 6 up, 7 in 2024-04-05T03:00:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:30 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:31 smithi033 ceph-mon[29244]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:31 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/1867141239' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:31 smithi033 ceph-mon[40008]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:31 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/1867141239' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:31 smithi151 ceph-mon[32789]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:31 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/1867141239' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:33.492 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:33 smithi151 ceph-mon[32789]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:33 smithi033 ceph-mon[29244]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:33 smithi033 ceph-mon[40008]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:35 smithi151 ceph-mon[32789]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:35.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:35 smithi033 ceph-mon[40008]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:35.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:35 smithi033 ceph-mon[29244]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:36.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:00:36.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:36.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:00:36.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:36.731 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:00:36.731 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:37.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:37 smithi151 ceph-mon[32789]: Deploying daemon osd.6 on smithi151 2024-04-05T03:00:37.717 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:37 smithi151 ceph-mon[32789]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:37 smithi033 ceph-mon[40008]: Deploying daemon osd.6 on smithi151 2024-04-05T03:00:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:37 smithi033 ceph-mon[40008]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:37.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:37 smithi033 ceph-mon[29244]: Deploying daemon osd.6 on smithi151 2024-04-05T03:00:37.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:37 smithi033 ceph-mon[29244]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:39.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:39 smithi033 ceph-mon[29244]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:39.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:39 smithi033 ceph-mon[40008]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:39.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:39 smithi151 ceph-mon[32789]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:41.579 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:41 smithi151 ceph-mon[32789]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:41.580 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:41 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[40008]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[29244]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:41 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:43 smithi033 ceph-mon[29244]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:43 smithi033 ceph-mon[40008]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:43 smithi151 ceph-mon[32789]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:44.565 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 6 on host 'smithi151' 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T03:00:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:44.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:44 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:44.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:44.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:44 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:45.159 DEBUG:teuthology.orchestra.run.smithi151:osd.6> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.6.service 2024-04-05T03:00:45.161 INFO:tasks.cephadm:Deploying osd.7 on smithi151 with /dev/vg_nvme/lv_1... 2024-04-05T03:00:45.162 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-05T03:00:45.921 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:00:45.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[29244]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[29244]: osdmap e45: 7 total, 6 up, 7 in 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[29244]: from='osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[29244]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[40008]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[40008]: osdmap e45: 7 total, 6 up, 7 in 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[40008]: from='osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:45.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:45 smithi033 ceph-mon[40008]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:46.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:45 smithi151 ceph-mon[32789]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-05T03:00:46.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:45 smithi151 ceph-mon[32789]: osdmap e45: 7 total, 6 up, 7 in 2024-04-05T03:00:46.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:45 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:46.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:45 smithi151 ceph-mon[32789]: from='osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:46.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:45 smithi151 ceph-mon[32789]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:00:46.028 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:00:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:00:45.733+0000 7f144f33b700 -1 osd.6 0 waiting for initial osdmap 2024-04-05T03:00:46.029 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:00:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:00:45.739+0000 7f1448933700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:00:47.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:46 smithi151 ceph-mon[32789]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:00:47.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:46 smithi151 ceph-mon[32789]: osdmap e46: 7 total, 6 up, 7 in 2024-04-05T03:00:47.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:47.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:46 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:47.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:46 smithi151 ceph-mon[32789]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:47.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[29244]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[29244]: osdmap e46: 7 total, 6 up, 7 in 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[29244]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[40008]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[40008]: osdmap e46: 7 total, 6 up, 7 in 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:47.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:46 smithi033 ceph-mon[40008]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T03:00:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:47 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T03:00:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:47 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T03:00:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:47 smithi151 ceph-mon[32789]: osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903] boot 2024-04-05T03:00:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:47 smithi151 ceph-mon[32789]: osdmap e47: 7 total, 7 up, 7 in 2024-04-05T03:00:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:47 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:47 smithi151 ceph-mon[32789]: osdmap e48: 7 total, 7 up, 7 in 2024-04-05T03:00:48.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T03:00:48.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T03:00:48.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[40008]: osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903] boot 2024-04-05T03:00:48.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[40008]: osdmap e47: 7 total, 7 up, 7 in 2024-04-05T03:00:48.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:48.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[40008]: osdmap e48: 7 total, 7 up, 7 in 2024-04-05T03:00:48.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T03:00:48.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T03:00:48.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[29244]: osd.6 [v2:172.21.15.151:6816/658137903,v1:172.21.15.151:6817/658137903] boot 2024-04-05T03:00:48.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[29244]: osdmap e47: 7 total, 7 up, 7 in 2024-04-05T03:00:48.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:00:48.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:47 smithi033 ceph-mon[29244]: osdmap e48: 7 total, 7 up, 7 in 2024-04-05T03:00:48.719 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-05T03:00:48.719 INFO:teuthology.orchestra.run.smithi151.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-05T03:00:48.719 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 10+0 records in 2024-04-05T03:00:48.719 INFO:teuthology.orchestra.run.smithi151.stderr:10+0 records out 2024-04-05T03:00:48.719 INFO:teuthology.orchestra.run.smithi151.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0128419 s, 817 MB/s 2024-04-05T03:00:48.719 INFO:teuthology.orchestra.run.smithi151.stderr: stderr: 2024-04-05T03:00:48.719 INFO:teuthology.orchestra.run.smithi151.stderr:--> Zapping successful for: 2024-04-05T03:00:49.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:48 smithi151 ceph-mon[32789]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:00:49.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:48 smithi033 ceph-mon[29244]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:00:49.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:48 smithi033 ceph-mon[40008]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:00:49.529 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi151:vg_nvme/lv_1 2024-04-05T03:00:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:49 smithi151 ceph-mon[32789]: osdmap e49: 7 total, 7 up, 7 in 2024-04-05T03:00:50.115 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:00:50.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:49 smithi033 ceph-mon[29244]: osdmap e49: 7 total, 7 up, 7 in 2024-04-05T03:00:50.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:49 smithi033 ceph-mon[40008]: osdmap e49: 7 total, 7 up, 7 in 2024-04-05T03:00:51.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: osdmap e50: 7 total, 7 up, 7 in 2024-04-05T03:00:51.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi151 2024-04-05T03:00:51.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi151 to 5696M 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:50 smithi151 ceph-mon[32789]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:00:51.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: osdmap e50: 7 total, 7 up, 7 in 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi151 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi151 to 5696M 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[29244]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: osdmap e50: 7 total, 7 up, 7 in 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi151 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi151 to 5696M 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:00:51.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:50 smithi033 ceph-mon[40008]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:00:52.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:51 smithi151 ceph-mon[32789]: osdmap e51: 7 total, 7 up, 7 in 2024-04-05T03:00:52.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:51 smithi033 ceph-mon[29244]: osdmap e51: 7 total, 7 up, 7 in 2024-04-05T03:00:52.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:51 smithi033 ceph-mon[40008]: osdmap e51: 7 total, 7 up, 7 in 2024-04-05T03:00:52.880 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:52 smithi151 ceph-mon[32789]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:52.880 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:52.880 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:52.881 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:52 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:52.881 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:52 smithi151 ceph-mon[32789]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[29244]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[29244]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[40008]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi151:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:53.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:52 smithi033 ceph-mon[40008]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-04-05T03:00:54.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/1209050893' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]: dispatch 2024-04-05T03:00:54.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]: dispatch 2024-04-05T03:00:54.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[29244]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]': finished 2024-04-05T03:00:54.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[29244]: osdmap e52: 8 total, 7 up, 8 in 2024-04-05T03:00:54.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:00:54.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/1209050893' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]: dispatch 2024-04-05T03:00:54.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]: dispatch 2024-04-05T03:00:54.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[40008]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]': finished 2024-04-05T03:00:54.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[40008]: osdmap e52: 8 total, 7 up, 8 in 2024-04-05T03:00:54.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:54 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:00:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:54 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/1209050893' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]: dispatch 2024-04-05T03:00:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:54 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]: dispatch 2024-04-05T03:00:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:54 smithi151 ceph-mon[32789]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a4bb8ad-d8b9-40cc-b066-429d1aba4543"}]': finished 2024-04-05T03:00:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:54 smithi151 ceph-mon[32789]: osdmap e52: 8 total, 7 up, 8 in 2024-04-05T03:00:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:54 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:00:55.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:55 smithi033 ceph-mon[29244]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:00:55.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:55 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/1425041247' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:55.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:55 smithi033 ceph-mon[40008]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:00:55.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:55 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/1425041247' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:55.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:55 smithi151 ceph-mon[32789]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:00:55.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:55 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/1425041247' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T03:00:57.514 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:57 smithi151 ceph-mon[32789]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-05T03:00:57.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:57 smithi033 ceph-mon[29244]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-05T03:00:57.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:57 smithi033 ceph-mon[40008]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-05T03:00:59.691 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:00:59 smithi151 ceph-mon[32789]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T03:00:59.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:00:59 smithi033 ceph-mon[40008]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T03:00:59.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:00:59 smithi033 ceph-mon[29244]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[40008]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[40008]: Deploying daemon osd.7 on smithi151 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[29244]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:01.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:01 smithi033 ceph-mon[29244]: Deploying daemon osd.7 on smithi151 2024-04-05T03:01:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:01 smithi151 ceph-mon[32789]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-05T03:01:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:01 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:01:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:01 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:01 smithi151 ceph-mon[32789]: Deploying daemon osd.7 on smithi151 2024-04-05T03:01:03.711 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:03 smithi151 ceph-mon[32789]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:03.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:03 smithi033 ceph-mon[40008]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:03.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:03 smithi033 ceph-mon[29244]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:05.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:05 smithi033 ceph-mon[29244]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:05.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:05 smithi033 ceph-mon[40008]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:05.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:05 smithi151 ceph-mon[32789]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:06.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:06 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:06.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:06 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:06.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:06 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:06.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:06 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:06.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:06 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:06.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:06 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:06.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:06 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:06.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:06 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:06.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:06 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:07.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:07 smithi151 ceph-mon[32789]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:07.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:07 smithi033 ceph-mon[29244]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:07.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:07 smithi033 ceph-mon[40008]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:08.987 INFO:teuthology.orchestra.run.smithi151.stdout:Created osd(s) 7 on host 'smithi151' 2024-04-05T03:01:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T03:01:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T03:01:09.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:09.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:09.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:09 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:09.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:09 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:09.537 DEBUG:teuthology.orchestra.run.smithi151:osd.7> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.7.service 2024-04-05T03:01:09.540 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-05T03:01:09.540 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd stat -f json 2024-04-05T03:01:09.812 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:10.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[29244]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-05T03:01:10.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[29244]: osdmap e53: 8 total, 7 up, 8 in 2024-04-05T03:01:10.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:10.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[29244]: from='osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:01:10.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[29244]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:01:10.196 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[40008]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-05T03:01:10.196 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[40008]: osdmap e53: 8 total, 7 up, 8 in 2024-04-05T03:01:10.197 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:10.197 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[40008]: from='osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:01:10.197 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:10 smithi033 ceph-mon[40008]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:01:10.505 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:10 smithi151 ceph-mon[32789]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-05T03:01:10.505 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:10 smithi151 ceph-mon[32789]: osdmap e53: 8 total, 7 up, 8 in 2024-04-05T03:01:10.505 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:10 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:10.505 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:10 smithi151 ceph-mon[32789]: from='osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:01:10.505 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:10 smithi151 ceph-mon[32789]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]: dispatch 2024-04-05T03:01:10.506 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:01:10 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:01:10.157+0000 7ff0d31b9700 -1 osd.7 0 waiting for initial osdmap 2024-04-05T03:01:10.506 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:01:10 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:01:10.166+0000 7ff0ccfb2700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:01:11.408 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:01:11.408 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: osdmap e54: 8 total, 7 up, 8 in 2024-04-05T03:01:11.408 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.408 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861] boot 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: osdmap e55: 8 total, 8 up, 8 in 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: osdmap e54: 8 total, 7 up, 8 in 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861] boot 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: osdmap e55: 8 total, 8 up, 8 in 2024-04-05T03:01:11.409 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:11 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.498 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi151", "root=default"]}]': finished 2024-04-05T03:01:11.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: osdmap e54: 8 total, 7 up, 8 in 2024-04-05T03:01:11.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T03:01:11.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: osd.7 [v2:172.21.15.151:6824/2209750861,v1:172.21.15.151:6825/2209750861] boot 2024-04-05T03:01:11.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: osdmap e55: 8 total, 8 up, 8 in 2024-04-05T03:01:11.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:11 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:11.837 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:01:12.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[29244]: purged_snaps scrub starts 2024-04-05T03:01:12.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[29244]: purged_snaps scrub ok 2024-04-05T03:01:12.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2864353735' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-05T03:01:12.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[29244]: osdmap e56: 8 total, 8 up, 8 in 2024-04-05T03:01:12.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[40008]: purged_snaps scrub starts 2024-04-05T03:01:12.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[40008]: purged_snaps scrub ok 2024-04-05T03:01:12.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2864353735' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-05T03:01:12.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:12 smithi033 ceph-mon[40008]: osdmap e56: 8 total, 8 up, 8 in 2024-04-05T03:01:12.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:12 smithi151 ceph-mon[32789]: purged_snaps scrub starts 2024-04-05T03:01:12.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:12 smithi151 ceph-mon[32789]: purged_snaps scrub ok 2024-04-05T03:01:12.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:12 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2864353735' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-05T03:01:12.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:12 smithi151 ceph-mon[32789]: osdmap e56: 8 total, 8 up, 8 in 2024-04-05T03:01:12.659 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1712286071,"num_in_osds":8,"osd_in_since":1712286054,"num_remapped_pgs":1} 2024-04-05T03:01:12.660 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:01:12.929 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:13.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: Detected new or changed devices on smithi151 2024-04-05T03:01:13.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi151 to 4272M 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[29244]: osdmap e57: 8 total, 8 up, 8 in 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: Detected new or changed devices on smithi151 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi151 to 4272M 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:13.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:13 smithi033 ceph-mon[40008]: osdmap e57: 8 total, 8 up, 8 in 2024-04-05T03:01:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: Detected new or changed devices on smithi151 2024-04-05T03:01:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi151 to 4272M 2024-04-05T03:01:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:13.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:13 smithi151 ceph-mon[32789]: osdmap e57: 8 total, 8 up, 8 in 2024-04-05T03:01:14.852 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:01:14.853 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":59,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:01:14.828146+0000","last_up_change":"2024-04-05T03:01:11.145768+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6803","nonce":122630893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6805","nonce":122630893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6809","nonce":122630893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6807","nonce":122630893}]},"public_addr":"172.21.15.33:6803/122630893","cluster_addr":"172.21.15.33:6805/122630893","heartbeat_back_addr":"172.21.15.33:6809/122630893","heartbeat_front_addr":"172.21.15.33:6807/122630893","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6811","nonce":2920348148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6813","nonce":2920348148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6817","nonce":2920348148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6815","nonce":2920348148}]},"public_addr":"172.21.15.33:6811/2920348148","cluster_addr":"172.21.15.33:6813/2920348148","heartbeat_back_addr":"172.21.15.33:6817/2920348148","heartbeat_front_addr":"172.21.15.33:6815/2920348148","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6819","nonce":3206964518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6821","nonce":3206964518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6825","nonce":3206964518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6823","nonce":3206964518}]},"public_addr":"172.21.15.33:6819/3206964518","cluster_addr":"172.21.15.33:6821/3206964518","heartbeat_back_addr":"172.21.15.33:6825/3206964518","heartbeat_front_addr":"172.21.15.33:6823/3206964518","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6827","nonce":1608936270}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6829","nonce":1608936270}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6833","nonce":1608936270}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6831","nonce":1608936270}]},"public_addr":"172.21.15.33:6827/1608936270","cluster_addr":"172.21.15.33:6829/1608936270","heartbeat_back_addr":"172.21.15.33:6833/1608936270","heartbeat_front_addr":"172.21.15.33:6831/1608936270","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:01:14.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:14 smithi033 ceph-mon[29244]: osdmap e58: 8 total, 8 up, 8 in 2024-04-05T03:01:14.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:14 smithi033 ceph-mon[29244]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:14.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:14 smithi033 ceph-mon[40008]: osdmap e58: 8 total, 8 up, 8 in 2024-04-05T03:01:14.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:14 smithi033 ceph-mon[40008]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:15.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:14 smithi151 ceph-mon[32789]: osdmap e58: 8 total, 8 up, 8 in 2024-04-05T03:01:15.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:14 smithi151 ceph-mon[32789]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:15.543 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-05T02:59:14.558935+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-05T03:01:15.544 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-05T03:01:15.806 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:16.177 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:15 smithi033 ceph-mon[40008]: osdmap e59: 8 total, 8 up, 8 in 2024-04-05T03:01:16.177 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:15 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1343622289' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:16.178 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:15 smithi033 ceph-mon[29244]: osdmap e59: 8 total, 8 up, 8 in 2024-04-05T03:01:16.178 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:15 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1343622289' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:16.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:15 smithi151 ceph-mon[32789]: osdmap e59: 8 total, 8 up, 8 in 2024-04-05T03:01:16.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:15 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1343622289' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:16.941 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:16 smithi033 ceph-mon[29244]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s, 0 objects/s recovering 2024-04-05T03:01:16.941 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:16 smithi033 ceph-mon[40008]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s, 0 objects/s recovering 2024-04-05T03:01:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:16 smithi151 ceph-mon[32789]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s, 0 objects/s recovering 2024-04-05T03:01:17.887 INFO:teuthology.orchestra.run.smithi033.stdout:pg_num: 1 2024-04-05T03:01:18.191 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:17 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2876921662' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-05T03:01:18.191 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:17 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2876921662' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-05T03:01:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:17 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2876921662' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-05T03:01:18.589 INFO:tasks.cephadm:Adding prometheus.a on smithi151 2024-04-05T03:01:18.590 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch apply prometheus '1;smithi151=a' 2024-04-05T03:01:18.852 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:01:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:18 smithi033 ceph-mon[29244]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:01:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:18 smithi033 ceph-mon[40008]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:01:19.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:18 smithi151 ceph-mon[32789]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T03:01:20.530 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled prometheus update... 2024-04-05T03:01:21.190 DEBUG:teuthology.orchestra.run.smithi151:prometheus.a> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service 2024-04-05T03:01:21.192 INFO:tasks.cephadm:Adding node-exporter.a on smithi033 2024-04-05T03:01:21.193 INFO:tasks.cephadm:Adding node-exporter.b on smithi151 2024-04-05T03:01:21.193 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch apply node-exporter '2;smithi033=a;smithi151=b' 2024-04-05T03:01:21.505 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: Saving service prometheus spec with placement smithi151=a;count:1 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:21.870 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:21 smithi151 ceph-mon[32789]: Deploying daemon prometheus.a on smithi151 2024-04-05T03:01:21.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-05T03:01:21.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: Saving service prometheus spec with placement smithi151=a;count:1 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[29244]: Deploying daemon prometheus.a on smithi151 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: Saving service prometheus spec with placement smithi151=a;count:1 2024-04-05T03:01:21.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:21.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:21.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:21.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:21.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:21.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:21 smithi033 ceph-mon[40008]: Deploying daemon prometheus.a on smithi151 2024-04-05T03:01:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:23 smithi151 ceph-mon[32789]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-05T03:01:23.891 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled node-exporter update... 2024-04-05T03:01:23.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:23 smithi033 ceph-mon[29244]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-05T03:01:23.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:23 smithi033 ceph-mon[40008]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-05T03:01:24.912 DEBUG:teuthology.orchestra.run.smithi033:node-exporter.a> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.a.service 2024-04-05T03:01:24.915 DEBUG:teuthology.orchestra.run.smithi151:node-exporter.b> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.b.service 2024-04-05T03:01:24.918 INFO:tasks.cephadm:Adding alertmanager.a on smithi033 2024-04-05T03:01:24.918 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch apply alertmanager '1;smithi033=a' 2024-04-05T03:01:25.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[40008]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi033=a;smithi151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[40008]: Saving service node-exporter spec with placement smithi033=a;smithi151=b;count:2 2024-04-05T03:01:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[40008]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:01:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[29244]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi033=a;smithi151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[29244]: Saving service node-exporter spec with placement smithi033=a;smithi151=b;count:2 2024-04-05T03:01:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:24 smithi033 ceph-mon[29244]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:01:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:24 smithi151 ceph-mon[32789]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi033=a;smithi151=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:24 smithi151 ceph-mon[32789]: Saving service node-exporter spec with placement smithi033=a;smithi151=b;count:2 2024-04-05T03:01:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:24 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:24 smithi151 ceph-mon[32789]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T03:01:25.668 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:01:27.522 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:27 smithi151 ceph-mon[32789]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-05T03:01:27.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:27 smithi033 ceph-mon[29244]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-05T03:01:27.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:27 smithi033 ceph-mon[40008]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-05T03:01:27.855 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled alertmanager update... 2024-04-05T03:01:29.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:28 smithi151 ceph-mon[32789]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi033=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:29.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:28 smithi151 ceph-mon[32789]: Saving service alertmanager spec with placement smithi033=a;count:1 2024-04-05T03:01:29.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:28 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:29.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:28 smithi151 ceph-mon[32789]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[29244]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi033=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[29244]: Saving service alertmanager spec with placement smithi033=a;count:1 2024-04-05T03:01:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[29244]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[40008]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi033=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[40008]: Saving service alertmanager spec with placement smithi033=a;count:1 2024-04-05T03:01:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:29.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:28 smithi033 ceph-mon[40008]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:29.294 DEBUG:teuthology.orchestra.run.smithi033:alertmanager.a> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@alertmanager.a.service 2024-04-05T03:01:29.296 INFO:tasks.cephadm:Adding grafana.a on smithi151 2024-04-05T03:01:29.296 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph orch apply grafana '1;smithi151=a' 2024-04-05T03:01:29.550 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:01:31.616 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:31 smithi151 ceph-mon[32789]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:31 smithi033 ceph-mon[29244]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:31.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:31 smithi033 ceph-mon[40008]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:33.224 INFO:teuthology.orchestra.run.smithi151.stdout:Scheduled grafana update... 2024-04-05T03:01:33.416 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:33 smithi151 ceph-mon[32789]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:33.416 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:33 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:33.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:33 smithi033 ceph-mon[40008]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:33 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:33.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:33 smithi033 ceph-mon[29244]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:33.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:33 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:34.023 DEBUG:teuthology.orchestra.run.smithi151:grafana.a> sudo journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service 2024-04-05T03:01:34.026 INFO:tasks.cephadm:Setting up client nodes... 2024-04-05T03:01:34.026 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-05T03:01:34.296 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:34.646 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:34 smithi033 ceph-mon[29244]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:34.646 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:34 smithi033 ceph-mon[29244]: Saving service grafana spec with placement smithi151=a;count:1 2024-04-05T03:01:34.647 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:34 smithi033 ceph-mon[40008]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:34.647 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:34 smithi033 ceph-mon[40008]: Saving service grafana spec with placement smithi151=a;count:1 2024-04-05T03:01:34.753 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:34 smithi151 ceph-mon[32789]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi151=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:01:34.753 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:34 smithi151 ceph-mon[32789]: Saving service grafana spec with placement smithi151=a;count:1 2024-04-05T03:01:35.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:34 smithi151 systemd[1]: Starting Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:01:35.415 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-mon[32789]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:35.415 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 podman[60121]: 2024-04-05 03:01:35.048372316 +0000 UTC m=+0.018569906 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T03:01:35.415 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 podman[60121]: 2024-04-05 03:01:35.170282358 +0000 UTC m=+0.140479928 container create 363364dc408900bf698c5386ca275474eac0f6a4ff2701fb2ab8b3ebf51e5805 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:01:35.415 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 podman[60121]: 2024-04-05 03:01:35.389144236 +0000 UTC m=+0.359341819 container init 363364dc408900bf698c5386ca275474eac0f6a4ff2701fb2ab8b3ebf51e5805 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:01:35.415 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 podman[60121]: 2024-04-05 03:01:35.391737045 +0000 UTC m=+0.361934617 container start 363364dc408900bf698c5386ca275474eac0f6a4ff2701fb2ab8b3ebf51e5805 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:01:35.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:35 smithi033 ceph-mon[29244]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:35.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:35 smithi033 ceph-mon[40008]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:35.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.415Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.415Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.415Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi151 (none))" 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.415Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.415Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.417Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.417Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.418Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.418Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.419Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.419Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.502µs 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.419Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T03:01:35.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.420Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-05T03:01:35.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.420Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.378µs wal_replay_duration=1.056127ms wbl_replay_duration=124ns total_replay_duration=1.089878ms 2024-04-05T03:01:35.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.421Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T03:01:35.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.421Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T03:01:35.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.421Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T03:01:35.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.435Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.943057ms db_storage=693ns remote_storage=1.071µs web_handler=292ns query_engine=647ns scrape=387.19µs scrape_sd=17.387µs notify=809ns notify_sd=1.723µs rules=13.374928ms tracing=6.628µs 2024-04-05T03:01:35.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.435Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T03:01:35.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:01:35.435Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T03:01:36.112 INFO:teuthology.orchestra.run.smithi033.stdout:[client.0] 2024-04-05T03:01:36.112 INFO:teuthology.orchestra.run.smithi033.stdout: key = AQCQaQ9m+CJjBhAAg3E4ayzL9/wxrk4Lk/d4nQ== 2024-04-05T03:01:36.350 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:36 smithi151 bash[60121]: 363364dc408900bf698c5386ca275474eac0f6a4ff2701fb2ab8b3ebf51e5805 2024-04-05T03:01:36.350 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:01:36 smithi151 systemd[1]: Started Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2199231284' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[29244]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[29244]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2199231284' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:36.459 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[40008]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:36.460 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[40008]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T03:01:36.460 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.460 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.460 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.460 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:36 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-05T03:01:36.766 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T03:01:36.766 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-05T03:01:36.766 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-05T03:01:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:36 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2199231284' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:36 smithi151 ceph-mon[32789]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:36 smithi151 ceph-mon[32789]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T03:01:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' 2024-04-05T03:01:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:36 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-05T03:01:36.809 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-05T03:01:37.075 INFO:teuthology.orchestra.run.smithi151.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T03:01:37.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-mon[29244]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-mon[29244]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-mon[29244]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-mon[40008]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-mon[40008]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-mon[40008]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setuser ceph since I am not root 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setgroup ceph since I am not root 2024-04-05T03:01:37.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-mgr[29477]: -- 172.21.15.33:0/3704104563 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56244518c9c0 con 0x562445162800 2024-04-05T03:01:37.511 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-mon[32789]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:37.512 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-mon[32789]: from='mgr.14152 172.21.15.33:0/3161865815' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-05T03:01:37.512 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-mon[32789]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-05T03:01:37.512 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ignoring --setuser ceph since I am not root 2024-04-05T03:01:37.512 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ignoring --setgroup ceph since I am not root 2024-04-05T03:01:37.512 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-mgr[34672]: -- 172.21.15.151:0/108195183 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c324f669c0 con 0x55c324f3d000 2024-04-05T03:01:37.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:37.728+0000 7f69d865d1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:01:38.140 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:37.820+0000 7f69d865d1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:01:38.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:37.733+0000 7fa7ced191c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:01:38.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:37.825+0000 7fa7ced191c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:01:38.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:38.466+0000 7fa7ced191c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:01:38.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:38.558+0000 7fa7ced191c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:01:38.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:38.642+0000 7fa7ced191c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:01:38.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:38.458+0000 7f69d865d1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:01:38.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:38.548+0000 7f69d865d1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:01:38.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:38.634+0000 7f69d865d1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:01:38.918 INFO:teuthology.orchestra.run.smithi151.stdout:[client.1] 2024-04-05T03:01:38.918 INFO:teuthology.orchestra.run.smithi151.stdout: key = AQCSaQ9moGGKNhAAEgA/MPo9tPC2KU0EWmDLQg== 2024-04-05T03:01:39.042 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-mon[32789]: from='client.? 172.21.15.151:0/2074589151' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:39.042 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-mon[32789]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:39.043 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-mon[32789]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T03:01:39.043 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:38.786+0000 7f69d865d1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:01:39.043 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:38.867+0000 7f69d865d1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:01:39.043 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:39.042+0000 7f69d865d1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:01:39.053 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-mon[29244]: from='client.? 172.21.15.151:0/2074589151' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:39.053 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-mon[29244]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:39.053 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-mon[29244]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T03:01:39.054 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-mon[40008]: from='client.? 172.21.15.151:0/2074589151' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:39.054 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-mon[40008]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T03:01:39.054 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-mon[40008]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T03:01:39.054 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:38.794+0000 7fa7ced191c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:01:39.054 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:38.876+0000 7fa7ced191c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:01:39.308 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:39.224+0000 7f69d865d1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:01:39.321 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:39 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:39.053+0000 7fa7ced191c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:01:39.321 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:39 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:39.236+0000 7fa7ced191c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:01:39.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:39 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:39.321+0000 7fa7ced191c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:01:39.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:39 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:39.415+0000 7fa7ced191c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:01:39.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:39 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:39.502+0000 7fa7ced191c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:01:39.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:39.308+0000 7f69d865d1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:01:39.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:39.401+0000 7f69d865d1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:01:39.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:39.487+0000 7f69d865d1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:01:39.848 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-05T03:01:39.848 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-05T03:01:39.848 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-05T03:01:39.892 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-05T03:01:39.892 DEBUG:teuthology.orchestra.run.smithi033:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-05T03:01:40.202 INFO:teuthology.misc.health.smithi033.stdout: 2024-04-05T03:01:40.203 INFO:teuthology.misc.health.smithi033.stdout:{"epoch":59,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:01:14.828146+0000","last_up_change":"2024-04-05T03:01:11.145768+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6803","nonce":122630893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6805","nonce":122630893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6809","nonce":122630893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6807","nonce":122630893}]},"public_addr":"172.21.15.33:6803/122630893","cluster_addr":"172.21.15.33:6805/122630893","heartbeat_back_addr":"172.21.15.33:6809/122630893","heartbeat_front_addr":"172.21.15.33:6807/122630893","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6811","nonce":2920348148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6813","nonce":2920348148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6817","nonce":2920348148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6815","nonce":2920348148}]},"public_addr":"172.21.15.33:6811/2920348148","cluster_addr":"172.21.15.33:6813/2920348148","heartbeat_back_addr":"172.21.15.33:6817/2920348148","heartbeat_front_addr":"172.21.15.33:6815/2920348148","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6819","nonce":3206964518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6821","nonce":3206964518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6825","nonce":3206964518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6823","nonce":3206964518}]},"public_addr":"172.21.15.33:6819/3206964518","cluster_addr":"172.21.15.33:6821/3206964518","heartbeat_back_addr":"172.21.15.33:6825/3206964518","heartbeat_front_addr":"172.21.15.33:6823/3206964518","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6827","nonce":1608936270}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6829","nonce":1608936270}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6833","nonce":1608936270}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6831","nonce":1608936270}]},"public_addr":"172.21.15.33:6827/1608936270","cluster_addr":"172.21.15.33:6829/1608936270","heartbeat_back_addr":"172.21.15.33:6833/1608936270","heartbeat_front_addr":"172.21.15.33:6831/1608936270","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:01:40.214 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-05T03:01:40.214 INFO:tasks.cephadm:Creating RBD pool 2024-04-05T03:01:40.215 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-05T03:01:40.484 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:40.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:40 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2752869245' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:40.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:40 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2752869245' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:40.595 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:40 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2752869245' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:40.852 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:40 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:40.535+0000 7f69d865d1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:01:40.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:40 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:40.663+0000 7f69d865d1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:01:40.853 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:40 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:40.852+0000 7f69d865d1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:01:40.933 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:40.595+0000 7fa7ced191c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:01:40.933 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:40.731+0000 7fa7ced191c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:01:40.934 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:40.933+0000 7fa7ced191c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:01:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:41.165+0000 7fa7ced191c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:01:41.254 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:41.079+0000 7f69d865d1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:01:41.521 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:41.254+0000 7f69d865d1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:01:41.522 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:41.444+0000 7f69d865d1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:01:41.610 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:41.347+0000 7fa7ced191c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:01:41.610 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:41.533+0000 7fa7ced191c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:01:41.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:41.722+0000 7f69d865d1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:01:41.881 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:41.807+0000 7fa7ced191c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:01:42.219 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:41.811+0000 7f69d865d1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:01:42.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:41.899+0000 7fa7ced191c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:01:42.481 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:42.218+0000 7f69d865d1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:01:42.650 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:42 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:42.314+0000 7fa7ced191c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:01:42.652 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'rbd' created 2024-04-05T03:01:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:42 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2186940032' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T03:01:42.910 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:42 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2186940032' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T03:01:42.910 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:42 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2186940032' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T03:01:43.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:43.144+0000 7fa7ced191c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:01:43.278 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-05T03:01:43.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:43.007+0000 7f69d865d1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:01:43.551 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:43.616 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:43.300+0000 7f69d865d1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:01:43.652 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:43.460+0000 7fa7ced191c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:01:43.922 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2186940032' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-05T03:01:43.922 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-mon[32789]: osdmap e60: 8 total, 8 up, 8 in 2024-04-05T03:01:43.922 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-mon[32789]: osdmap e61: 8 total, 8 up, 8 in 2024-04-05T03:01:43.922 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:43.615+0000 7f69d865d1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:43.814+0000 7f69d865d1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: [05/Apr/2024:03:01:43] ENGINE Bus STARTING 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: CherryPy Checker: 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: The Application mounted at '' has an empty config. 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: [05/Apr/2024:03:01:43] ENGINE Serving on http://:::9283 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: [05/Apr/2024:03:01:43] ENGINE Bus STARTED 2024-04-05T03:01:43.923 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2186940032' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-05T03:01:43.924 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-mon[29244]: osdmap e60: 8 total, 8 up, 8 in 2024-04-05T03:01:43.924 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-mon[29244]: osdmap e61: 8 total, 8 up, 8 in 2024-04-05T03:01:43.924 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:43.798+0000 7fa7ced191c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:01:43.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2186940032' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-05T03:01:43.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-mon[40008]: osdmap e60: 8 total, 8 up, 8 in 2024-04-05T03:01:43.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:43 smithi033 ceph-mon[40008]: osdmap e61: 8 total, 8 up, 8 in 2024-04-05T03:01:44.202 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:44.017+0000 7fa7ced191c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:01:44.202 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:01:44] ENGINE Bus STARTING 2024-04-05T03:01:44.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: CherryPy Checker: 2024-04-05T03:01:44.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: The Application mounted at '' has an empty config. 2024-04-05T03:01:44.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:01:44.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:01:44] ENGINE Serving on http://:::9283 2024-04-05T03:01:44.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:01:44] ENGINE Bus STARTED 2024-04-05T03:01:44.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: Standby manager daemon x restarted 2024-04-05T03:01:44.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: Standby manager daemon x started 2024-04-05T03:01:44.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:01:44.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:01:44.911 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: Active manager daemon y restarted 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: Activating manager daemon y 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: osdmap e62: 8 total, 8 up, 8 in 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: mgrmap e17: y(active, starting, since 0.00972301s), standbys: x 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:01:44.912 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: Manager daemon y is now available 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:44.913 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:01:44.914 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:01:44.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: Standby manager daemon x restarted 2024-04-05T03:01:44.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: Standby manager daemon x started 2024-04-05T03:01:44.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:01:44.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:01:44.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:01:44.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:01:44.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: Active manager daemon y restarted 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: Activating manager daemon y 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: osdmap e62: 8 total, 8 up, 8 in 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: mgrmap e17: y(active, starting, since 0.00972301s), standbys: x 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:01:44.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:01:44.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: Manager daemon y is now available 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: Standby manager daemon x restarted 2024-04-05T03:01:44.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: Standby manager daemon x started 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/3338504631' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: Active manager daemon y restarted 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: Activating manager daemon y 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: osdmap e62: 8 total, 8 up, 8 in 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: mgrmap e17: y(active, starting, since 0.00972301s), standbys: x 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:01:44.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: Manager daemon y is now available 2024-04-05T03:01:44.934 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:44.935 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:44.935 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:44.935 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:44.935 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:01:44.935 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:01:46.037 INFO:teuthology.orchestra.run.smithi033.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-05T03:01:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:46 smithi151 ceph-mon[32789]: mgrmap e18: y(active, since 1.01174s), standbys: x 2024-04-05T03:01:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:46 smithi151 ceph-mon[32789]: pgmap v3: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:46 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2731556694' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:01:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:46 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:46.364 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[29244]: mgrmap e18: y(active, since 1.01174s), standbys: x 2024-04-05T03:01:46.365 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[29244]: pgmap v3: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:46.365 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2731556694' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:01:46.365 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:46.365 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[40008]: mgrmap e18: y(active, since 1.01174s), standbys: x 2024-04-05T03:01:46.365 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[40008]: pgmap v3: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:46.365 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2731556694' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:01:46.365 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:46 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:46.659 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-05T03:01:46.660 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-05T03:01:46.660 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-05T03:01:46.942 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:47.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:01:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:01:47.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: [05/Apr/2024:03:01:45] ENGINE Bus STARTING 2024-04-05T03:01:47.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: [05/Apr/2024:03:01:45] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:01:47.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: [05/Apr/2024:03:01:45] ENGINE Bus STARTED 2024-04-05T03:01:47.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2731556694' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: osdmap e63: 8 total, 8 up, 8 in 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.293 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: [05/Apr/2024:03:01:45] ENGINE Bus STARTING 2024-04-05T03:01:47.293 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: [05/Apr/2024:03:01:45] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:01:47.293 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: [05/Apr/2024:03:01:45] ENGINE Bus STARTED 2024-04-05T03:01:47.293 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:01:47.293 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2731556694' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: osdmap e63: 8 total, 8 up, 8 in 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.294 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: [05/Apr/2024:03:01:45] ENGINE Bus STARTING 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: [05/Apr/2024:03:01:45] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: [05/Apr/2024:03:01:45] ENGINE Bus STARTED 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2731556694' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: osdmap e63: 8 total, 8 up, 8 in 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-05T03:01:47.295 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:01:47.296 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:48.225 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:48 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi151 to 4016M 2024-04-05T03:01:48.225 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:48 smithi033 ceph-mon[29244]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:01:48.225 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:48 smithi033 ceph-mon[29244]: Cluster is now healthy 2024-04-05T03:01:48.225 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:48 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi151 to 4016M 2024-04-05T03:01:48.225 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:48 smithi033 ceph-mon[40008]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:01:48.226 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:48 smithi033 ceph-mon[40008]: Cluster is now healthy 2024-04-05T03:01:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:48 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi151 to 4016M 2024-04-05T03:01:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:48 smithi151 ceph-mon[32789]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:01:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:48 smithi151 ceph-mon[32789]: Cluster is now healthy 2024-04-05T03:01:48.727 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:01:49.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[40008]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[40008]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1517648045' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[29244]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[29244]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:49.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:49 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1517648045' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:01:49.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:49 smithi151 ceph-mon[32789]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:49.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:49 smithi151 ceph-mon[32789]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-05T03:01:49.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:49.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:49.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:49.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:49.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:49 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1517648045' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:01:49.547 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":20,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":2530405471},{"type":"v1","addr":"172.21.15.33:6801","nonce":2530405471}]},"active_addr":"172.21.15.33:6801/2530405471","active_change":"2024-04-05T03:01:44.019851+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.33:8443/","prometheus":"http://172.21.15.33:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":62,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":2236066571}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3439747338}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":1920159030}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3813037768}]}]} 2024-04-05T03:01:49.551 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-05T03:01:49.551 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-05T03:01:49.551 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:01:49.815 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:50 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:01:50.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:01:50.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:01:50.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:01:50.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:01:50.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:01:50.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:50.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:50 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[29244]: Deploying daemon alertmanager.a on smithi033 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[29244]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[40008]: Deploying daemon alertmanager.a on smithi033 2024-04-05T03:01:51.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:51 smithi033 ceph-mon[40008]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:51 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:01:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:51 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:01:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:51 smithi151 ceph-mon[32789]: Deploying daemon alertmanager.a on smithi033 2024-04-05T03:01:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:51 smithi151 ceph-mon[32789]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:01:51.656 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:01:51.656 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":63,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:01:46.026943+0000","last_up_change":"2024-04-05T03:01:11.145768+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T03:01:42.593775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6803","nonce":122630893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6805","nonce":122630893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6809","nonce":122630893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6807","nonce":122630893}]},"public_addr":"172.21.15.33:6803/122630893","cluster_addr":"172.21.15.33:6805/122630893","heartbeat_back_addr":"172.21.15.33:6809/122630893","heartbeat_front_addr":"172.21.15.33:6807/122630893","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6811","nonce":2920348148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6813","nonce":2920348148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6817","nonce":2920348148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6815","nonce":2920348148}]},"public_addr":"172.21.15.33:6811/2920348148","cluster_addr":"172.21.15.33:6813/2920348148","heartbeat_back_addr":"172.21.15.33:6817/2920348148","heartbeat_front_addr":"172.21.15.33:6815/2920348148","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6819","nonce":3206964518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6821","nonce":3206964518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6825","nonce":3206964518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6823","nonce":3206964518}]},"public_addr":"172.21.15.33:6819/3206964518","cluster_addr":"172.21.15.33:6821/3206964518","heartbeat_back_addr":"172.21.15.33:6825/3206964518","heartbeat_front_addr":"172.21.15.33:6823/3206964518","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6827","nonce":1608936270}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6829","nonce":1608936270}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6833","nonce":1608936270}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6831","nonce":1608936270}]},"public_addr":"172.21.15.33:6827/1608936270","cluster_addr":"172.21.15.33:6829/1608936270","heartbeat_back_addr":"172.21.15.33:6833/1608936270","heartbeat_front_addr":"172.21.15.33:6831/1608936270","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3239504883":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/2211132569":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/1272921316":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/1632976587":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:01:52.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:52 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1785384977' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:52.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:52 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1785384977' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:52.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:52 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1785384977' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:52.703 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-05T03:01:52.703 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:01:52.991 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:53.336 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:53 smithi033 ceph-mon[29244]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:01:53.336 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:53 smithi033 ceph-mon[40008]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:01:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:53 smithi151 ceph-mon[32789]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:01:55.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:55 smithi033 ceph-mon[29244]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:01:55.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:55 smithi033 ceph-mon[40008]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:01:55.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:55 smithi151 ceph-mon[32789]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:01:55.750 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:01:55.750 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":63,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:01:46.026943+0000","last_up_change":"2024-04-05T03:01:11.145768+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T03:01:42.593775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6803","nonce":122630893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6805","nonce":122630893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6809","nonce":122630893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6807","nonce":122630893}]},"public_addr":"172.21.15.33:6803/122630893","cluster_addr":"172.21.15.33:6805/122630893","heartbeat_back_addr":"172.21.15.33:6809/122630893","heartbeat_front_addr":"172.21.15.33:6807/122630893","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6811","nonce":2920348148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6813","nonce":2920348148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6817","nonce":2920348148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6815","nonce":2920348148}]},"public_addr":"172.21.15.33:6811/2920348148","cluster_addr":"172.21.15.33:6813/2920348148","heartbeat_back_addr":"172.21.15.33:6817/2920348148","heartbeat_front_addr":"172.21.15.33:6815/2920348148","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6819","nonce":3206964518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6821","nonce":3206964518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6825","nonce":3206964518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6823","nonce":3206964518}]},"public_addr":"172.21.15.33:6819/3206964518","cluster_addr":"172.21.15.33:6821/3206964518","heartbeat_back_addr":"172.21.15.33:6825/3206964518","heartbeat_front_addr":"172.21.15.33:6823/3206964518","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6827","nonce":1608936270}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6829","nonce":1608936270}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6833","nonce":1608936270}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6831","nonce":1608936270}]},"public_addr":"172.21.15.33:6827/1608936270","cluster_addr":"172.21.15.33:6829/1608936270","heartbeat_back_addr":"172.21.15.33:6833/1608936270","heartbeat_front_addr":"172.21.15.33:6831/1608936270","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3239504883":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/2211132569":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/1272921316":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/1632976587":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:01:56.414 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:56 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3371527745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:56 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3371527745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:56.440 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-05T03:01:56.441 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-05T03:01:56.441 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-05T03:01:56.441 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-05T03:01:56.442 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-05T03:01:56.442 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-05T03:01:56.443 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-05T03:01:56.444 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-05T03:01:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:56 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3371527745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:01:57.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:57 smithi033 ceph-mon[29244]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T03:01:57.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:57 smithi033 ceph-mon[40008]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T03:01:57.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:57 smithi151 ceph-mon[32789]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T03:01:57.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:01:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:01:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:01:59.099 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:59.105 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:59.165 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:59.197 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:01:59.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:01:59 smithi033 ceph-mon[29244]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:01:59.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:01:59 smithi033 ceph-mon[40008]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:01:59.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:01:59 smithi151 ceph-mon[32789]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:01:59.628 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:00.162 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:00 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:00.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:00 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:00.167 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:00.168 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:00.168 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:00.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:00 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:01.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:01 smithi033 ceph-mon[29244]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:02:01.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:01 smithi033 ceph-mon[40008]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:02:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:01 smithi151 ceph-mon[32789]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:02:03.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:03 smithi033 ceph-mon[29244]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:02:03.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:03 smithi033 ceph-mon[40008]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:02:03.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:03 smithi151 ceph-mon[32789]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:02:05.202 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:05 smithi033 ceph-mon[40008]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:05.203 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:05 smithi033 ceph-mon[29244]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:05.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:05 smithi151 ceph-mon[32789]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:07 smithi151 ceph-mon[32789]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:07.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:02:07 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:02:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:02:07.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:07 smithi033 ceph-mon[29244]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:07.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:07 smithi033 ceph-mon[40008]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:08.407 INFO:teuthology.orchestra.run.smithi033.stdout:167503724566 2024-04-05T03:02:08.407 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-05T03:02:08.665 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:08 smithi033 podman[70478]: 2024-04-05 03:02:08.666584393 +0000 UTC m=+1.131013920 volume create 282a33892dc5b01413b42a7441561962fc432839c06bad3c162abe8282faceff 2024-04-05T03:02:08.940 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:08 smithi033 podman[70478]: 2024-04-05 03:02:08.780654248 +0000 UTC m=+1.245083814 container create a83da33c8a66f1230c6361a5f561ba2a7935a4e40025c0fdddb1ba38b1d50f21 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:09.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:09 smithi033 ceph-mon[29244]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:09.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:09 smithi033 ceph-mon[40008]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:09.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:09 smithi151 ceph-mon[32789]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 podman[70478]: 2024-04-05 03:02:10.498600298 +0000 UTC m=+2.963029828 container init a83da33c8a66f1230c6361a5f561ba2a7935a4e40025c0fdddb1ba38b1d50f21 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 podman[70478]: 2024-04-05 03:02:10.502678288 +0000 UTC m=+2.967107813 container start a83da33c8a66f1230c6361a5f561ba2a7935a4e40025c0fdddb1ba38b1d50f21 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.514Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.514Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.516Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.33 port=9094 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.517Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.540Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.540Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.543Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-05T03:02:10.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:10.543Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-05T03:02:11.100 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-mon[29244]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:11.100 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-mon[40008]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:11.127 INFO:teuthology.orchestra.run.smithi033.stdout:201863462930 2024-04-05T03:02:11.128 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-05T03:02:11.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:11 smithi151 ceph-mon[32789]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:11.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:11 smithi033 bash[70478]: a83da33c8a66f1230c6361a5f561ba2a7935a4e40025c0fdddb1ba38b1d50f21 2024-04-05T03:02:11.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:11 smithi033 systemd[1]: Started Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:11.992 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:11] ENGINE Bus STOPPING 2024-04-05T03:02:11.992 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:02:11.992 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:11] ENGINE Bus STOPPED 2024-04-05T03:02:11.992 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:11] ENGINE Bus STARTING 2024-04-05T03:02:11.992 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:11] ENGINE Serving on http://:::9283 2024-04-05T03:02:11.992 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:11] ENGINE Bus STARTED 2024-04-05T03:02:12.582 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:12.517Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000527716s 2024-04-05T03:02:12.735 INFO:teuthology.orchestra.run.smithi033.stdout:77309411365 2024-04-05T03:02:12.735 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-05T03:02:12.901 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.901 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.901 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.901 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.901 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: Deploying daemon grafana.a on smithi151 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[29244]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.902 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T03:02:12.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T03:02:12.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:12.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: Deploying daemon grafana.a on smithi151 2024-04-05T03:02:12.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:12 smithi033 ceph-mon[40008]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:13.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:13.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T03:02:13.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T03:02:13.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:13.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: Deploying daemon grafana.a on smithi151 2024-04-05T03:02:13.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:12 smithi151 ceph-mon[32789]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:13.759 INFO:teuthology.orchestra.run.smithi033.stdout:34359738416 2024-04-05T03:02:13.760 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-05T03:02:14.219 INFO:teuthology.orchestra.run.smithi033.stdout:133143986203 2024-04-05T03:02:14.219 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-05T03:02:14.727 INFO:teuthology.orchestra.run.smithi033.stdout:111669149728 2024-04-05T03:02:14.727 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-05T03:02:14.736 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:14.736 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:15.220 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:15 smithi033 ceph-mon[29244]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:15.220 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:15 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:15.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:15 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:15.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:15 smithi033 ceph-mon[40008]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:15.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:15 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:15.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:15 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:15.523 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:15.523 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:15.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:15 smithi151 ceph-mon[32789]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:15 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:15 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:15.531 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:16.051 INFO:teuthology.orchestra.run.smithi033.stdout:236223201295 2024-04-05T03:02:16.052 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-05T03:02:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:17 smithi151 ceph-mon[32789]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:17.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:02:17 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:02:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:02:17.371 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:17 smithi033 ceph-mon[40008]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:17.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:17 smithi033 ceph-mon[29244]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:18.706 INFO:teuthology.orchestra.run.smithi033.stdout:34359738417 2024-04-05T03:02:19.337 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:19 smithi033 ceph-mon[40008]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:19.337 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:19 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1535199835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:02:19.337 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:19 smithi033 ceph-mon[29244]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:19.337 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:19 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1535199835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:02:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:19 smithi151 ceph-mon[32789]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:19 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1535199835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:02:20.700 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:20.520Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002792786s 2024-04-05T03:02:20.891 INFO:teuthology.orchestra.run.smithi033.stdout:55834574892 2024-04-05T03:02:20.891 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-05T03:02:21.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:21 smithi033 ceph-mon[29244]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:21.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:21 smithi033 ceph-mon[40008]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:21.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:21 smithi151 ceph-mon[32789]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:21.974 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738417 for osd.0 2024-04-05T03:02:21.974 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:22.376 INFO:teuthology.orchestra.run.smithi033.stdout:133143986206 2024-04-05T03:02:23.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:23 smithi033 ceph-mon[40008]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:23.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:23 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/662902488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:02:23.162 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:23 smithi033 ceph-mon[29244]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:23.162 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:23 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/662902488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:02:23.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:23 smithi151 ceph-mon[32789]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:23.355 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:23 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/662902488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:02:23.970 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986206 for osd.4 2024-04-05T03:02:23.970 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:23.995 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:24.653 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:24.654 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:25.201 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:25 smithi033 ceph-mon[40008]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:25.202 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:25 smithi033 ceph-mon[29244]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:25.306 INFO:teuthology.orchestra.run.smithi033.stdout:167503724569 2024-04-05T03:02:25.479 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:25 smithi151 ceph-mon[32789]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:25.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:25 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:02:26.188 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4178797452' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:02:26.188 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:25 smithi151 podman[62161]: 2024-04-05 03:02:25.822989403 +0000 UTC m=+0.016883176 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:02:26.188 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:25 smithi151 podman[62161]: 2024-04-05 03:02:25.945183765 +0000 UTC m=+0.139077539 container create a5ab46c212de9f98990643abd945eebb5928e359a2dc0d1a19ba9ca05d63628b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.buildah.version=1.32.0, name=ubi8, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-05T03:02:26.188 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 podman[62161]: 2024-04-05 03:02:26.189193693 +0000 UTC m=+0.383087465 container init a5ab46c212de9f98990643abd945eebb5928e359a2dc0d1a19ba9ca05d63628b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, release=239.1651231664, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, architecture=x86_64) 2024-04-05T03:02:26.396 INFO:teuthology.orchestra.run.smithi033.stdout:111669149732 2024-04-05T03:02:26.397 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:26 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4178797452' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:02:26.397 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:26 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4178797452' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 podman[62161]: 2024-04-05 03:02:26.192560701 +0000 UTC m=+0.386454493 container start a5ab46c212de9f98990643abd945eebb5928e359a2dc0d1a19ba9ca05d63628b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=ubi8, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.buildah.version=1.32.0) 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 bash[62161]: a5ab46c212de9f98990643abd945eebb5928e359a2dc0d1a19ba9ca05d63628b 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349377125Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349543439Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349560328Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.34956676Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349572524Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349578138Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-05T03:02:26.439 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349583446Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-05T03:02:26.440 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349589436Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-05T03:02:26.440 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349595594Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-05T03:02:26.440 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349600898Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-05T03:02:26.441 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349605915Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-05T03:02:26.441 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349610993Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-05T03:02:26.441 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349616641Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-05T03:02:26.441 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=settings t=2024-04-05T03:02:26.349622458Z level=info msg="App mode production" 2024-04-05T03:02:26.441 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=sqlstore t=2024-04-05T03:02:26.349670111Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-05T03:02:26.441 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=sqlstore t=2024-04-05T03:02:26.349686865Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.350537742Z level=info msg="Starting DB migrations" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.350633475Z level=info msg="Executing migration" id="create migration_log table" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.35118643Z level=info msg="Executing migration" id="create user table" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.35165539Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.352169275Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.352619892Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.353193844Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.353604812Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.354322972Z level=info msg="Executing migration" id="create user table v2" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.354752585Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.355184212Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.355615965Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-05T03:02:26.442 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.356003294Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.356393231Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.356806766Z level=info msg="Executing migration" id="Update user table charset" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.357237538Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.357879081Z level=info msg="Executing migration" id="Add missing user data" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.358223234Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.358606035Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.359007054Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.359416549Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.360549364Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.360985522Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.361463223Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.361880869Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-05T03:02:26.443 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.362302014Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.362719328Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.363036247Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.363437833Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.363813862Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.364227926Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.364621212Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.365293127Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.365711351Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.366138866Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.366552199Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.366966429Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-05T03:02:26.444 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.367395502Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.3677272Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.36810848Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.36846456Z level=info msg="Executing migration" id="create star table" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.368829472Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.369263937Z level=info msg="Executing migration" id="create org table v1" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.369668264Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.370104584Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.370507195Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.370926295Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.371364825Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.371755703Z level=info msg="Executing migration" id="Update org table charset" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.37209012Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-05T03:02:26.445 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.372384165Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.372680324Z level=info msg="Executing migration" id="create dashboard table" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.373827433Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.37423818Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.374652945Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.375091077Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.375881306Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.376686048Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.377531012Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.378343833Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.379244185Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.382170411Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.382675037Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.38309046Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-05T03:02:26.446 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.385792965Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.386422514Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.386887613Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.387413144Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.389554343Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.390099279Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.392315254Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.395026935Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.395424696Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.395696266Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.396224622Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.398899087Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.401615454Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.402197388Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.404958018Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-05T03:02:26.447 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.40753915Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.410183999Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.410513257Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.413485572Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.413907817Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.416170042Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.419021189Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.419613069Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.422448342Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.425060154Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.427564182Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.428112961Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.430882675Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.433573549Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-05T03:02:26.448 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.436322665Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-05T03:02:26.449 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.436755124Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-05T03:02:26.449 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.439991125Z level=info msg="Executing migration" id="create data_source table" 2024-04-05T03:02:26.449 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.442254617Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-05T03:02:26.690 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.445083364Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.445540072Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.448355654Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.451114755Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.453989244Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.454494485Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.455285031Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.45624354Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.457024567Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.457568231Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.458412409Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.45928943Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.460160276Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-05T03:02:26.691 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.460714929Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.461662594Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.462400637Z level=info msg="Executing migration" id="Add uid column" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.463362046Z level=info msg="Executing migration" id="Update uid value" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.463873496Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.464673012Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.465530766Z level=info msg="Executing migration" id="create api_key table" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.466368638Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.466824086Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.467644857Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.468496418Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.469440536Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.470308625Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.471197054Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-05T03:02:26.692 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.472265189Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.473138347Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.473983736Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.47495576Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.475695453Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.476564592Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.477543309Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.478284026Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.479295343Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.480032034Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-05T03:02:26.693 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.481004362Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-05T03:02:26.694 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.48176978Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-05T03:02:26.694 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.482600106Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-05T03:02:26.694 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.485522066Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.488543338Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.491192441Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.494158634Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.497005386Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.500025346Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.502826955Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.505701709Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.508599402Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.511553739Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.514455678Z level=info msg="Executing migration" id="create quota table v1" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.51742572Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.520313954Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.523347856Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-05T03:02:26.695 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.52615931Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.529152891Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.532027372Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.534793206Z level=info msg="Executing migration" id="create session table" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.53745901Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.540337916Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.543228424Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.546038975Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.548999226Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.551635504Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.554270701Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.556938392Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.559642521Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.562643431Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-05T03:02:26.696 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.565279192Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.568209727Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.571027344Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.573652237Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.576375172Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.57908202Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.581835639Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.582631105Z level=info msg="Executing migration" id="create alert table v1" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.583639487Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.584371422Z level=info msg="Executing migration" id="add index alert state" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.585347565Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.586102838Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.587053315Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.587770265Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.588732452Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-05T03:02:26.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.590140171Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.591046273Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.592043147Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.592851118Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.593594879Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.594531743Z level=info msg="Executing migration" id="Add column is_default" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.595328123Z level=info msg="Executing migration" id="Add column frequency" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.596257274Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.597483261Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.598229445Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.599089297Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.600028766Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-05T03:02:26.698 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.600826828Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.601865771Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.602687637Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.603451282Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.60427898Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.604705664Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.605355623Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.606240806Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.607202176Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.608126442Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.608916868Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.609765649Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.610649681Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.611510121Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.612535745Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-05T03:02:26.699 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.615148511Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.618375651Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.62126137Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.624082463Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.62457876Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.625050889Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.627726832Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.628048829Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.630405881Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.633053143Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.635608131Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.638051Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.640716124Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.641478084Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-05T03:02:26.700 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.643141418Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.643938757Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.644783642Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.645714082Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.646462074Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.647223104Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.647935224Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.648673927Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.649489207Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.650416731Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.651121488Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.651900162Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.652675084Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.653523Z level=info msg="Executing migration" id="Move region to single row" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.654308664Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.655195876Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-05T03:02:26.701 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.655897415Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.656912597Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.657537468Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.65850411Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.659237483Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.660076434Z level=info msg="Executing migration" id="create test_data table" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.660527849Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.660946517Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.661809426Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.662746124Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.663401354Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.664174389Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.664518708Z level=info msg="Executing migration" id="create team table" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.665093538Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-05T03:02:26.702 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.666036842Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.666758339Z level=info msg="Executing migration" id="create team member table" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.667558591Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.668358465Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.669291507Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.672196472Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.674769465Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.677369237Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.680157677Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.682793634Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.685558136Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.688276117Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.688777965Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.691342391Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-05T03:02:26.703 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.693952213Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-05T03:02:26.942 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.696743577Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-05T03:02:26.942 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.699315648Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.69973431Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.702379284Z level=info msg="Executing migration" id="create tag table" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.705116039Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.707852459Z level=info msg="Executing migration" id="create login attempt table" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.7083359Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.71115647Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.713670381Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.716514643Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.71692628Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.719550436Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.722097041Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.724832335Z level=info msg="Executing migration" id="create user auth table" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.725244977Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-05T03:02:26.943 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.727907219Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.730624669Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.73299878Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.733783297Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.736536082Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.738993786Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.741500735Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.742302871Z level=info msg="Executing migration" id="create server_lock table" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.744854636Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.747576023Z level=info msg="Executing migration" id="create user auth token table" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.750112216Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.750535752Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.751446677Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.752094167Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.752882467Z level=info msg="Executing migration" id="create cache_data table" 2024-04-05T03:02:26.944 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.753289301Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-05T03:02:26.945 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.754158946Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-05T03:02:26.945 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.754967559Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-05T03:02:26.945 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.755744846Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-05T03:02:26.945 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.756063223Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-05T03:02:26.945 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.756334959Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-05T03:02:26.945 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.756781457Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-05T03:02:26.945 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.757574128Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-05T03:02:26.946 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.758048344Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-05T03:02:26.946 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.758348917Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-05T03:02:26.946 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.758759635Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-05T03:02:26.946 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.759194107Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.759633274Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.760101804Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.760919744Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.761353002Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.761657974Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.76207474Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.762547671Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.762985412Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.763281033Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.763688838Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.764107503Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.764547124Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.76499716Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-05T03:02:26.947 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.765838385Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.766266061Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.766678805Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.770009213Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.773350868Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.773798941Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.774241887Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.775105192Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.775561236Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.776032524Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.776483673Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.776927474Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.777233327Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.77812302Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.778966068Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-05T03:02:26.948 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.779759851Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.780219365Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.780653418Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.781468567Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.782313328Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.782768477Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.783600285Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.784429865Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.784777393Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.785236367Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.785681482Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.788582575Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.791547909Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.794419542Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-05T03:02:26.949 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.795378201Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.798354249Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.801144793Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.804252096Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.804727227Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.807661357Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.810330099Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.813143118Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.814053688Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.816948862Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.819876262Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.822717782Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.823263019Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.826060216Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-05T03:02:26.950 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.828875647Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.831670943Z level=info msg="Executing migration" id="create alert_image table" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.83213574Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.834886549Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.83772367Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.840262381Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.840729786Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.840823495Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.843640546Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.846377666Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.849016541Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.849281636Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.849397451Z level=info msg="alerts found to migrate" alerts=0 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.849805531Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-05T03:02:26.951 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.852691195Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.855225408Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.858055421Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.858519427Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.861277075Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.863923624Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.866668606Z level=info msg="Executing migration" id="create data_keys table" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.867139042Z level=info msg="Executing migration" id="create secrets table" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.867968265Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.872226244Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.873196484Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.873517222Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.877733995Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.881901319Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-05T03:02:26.952 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.882731867Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.883315459Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.88422957Z level=info msg="Executing migration" id="create permission table" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.884996536Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.88586958Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.886336566Z level=info msg="Executing migration" id="create role table" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.887144822Z level=info msg="Executing migration" id="add column display_name" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.888086767Z level=info msg="Executing migration" id="add column group_name" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.889014538Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.889481382Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.89042589Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.891202364Z level=info msg="Executing migration" id="create team role table" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.892170214Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.892658583Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-05T03:02:26.953 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.893538937Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.894410367Z level=info msg="Executing migration" id="create user role table" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.895289248Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.895763469Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.896619992Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.897614902Z level=info msg="Executing migration" id="create builtin role table" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.898361185Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.898812847Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.899704532Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.900680355Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.901689278Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.902338691Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.903256998Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.903878165Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.904279882Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-05T03:02:26.954 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.907294182Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.908308265Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.911218881Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.914186032Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.916855041Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.917299195Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.920229067Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.92073829Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.923116922Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.925888157Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.926300672Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.929482963Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.932029478Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.934938088Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-05T03:02:26.955 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.937756947Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-05T03:02:26.956 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.940424598Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-05T03:02:26.956 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.94326092Z level=info msg="Executing migration" id="add correlation config column" 2024-04-05T03:02:26.956 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.945972049Z level=info msg="Executing migration" id="create entity_events table" 2024-04-05T03:02:26.956 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.946571115Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-05T03:02:27.263 INFO:teuthology.orchestra.run.smithi033.stdout:201863462934 2024-04-05T03:02:27.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:02:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:02:27.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-mon[32789]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:27.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2038977600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:02:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1671162758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:02:27.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.94937494Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-05T03:02:27.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.949496671Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-05T03:02:27.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.95218383Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.952279962Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.954956453Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.955369277Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.95807791Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.958519814Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.960744052Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.963478206Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.964015396Z level=info msg="Executing migration" id="Drop public config table" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.966884558Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.969598515Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.972431305Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.972995858Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.975771718Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.978761374Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-05T03:02:27.280 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.979768615Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.980777093Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.981759854Z level=info msg="Executing migration" id="add share column" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.982791122Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.9837456Z level=info msg="Executing migration" id="create file table" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.984397251Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.98523145Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.986066915Z level=info msg="Executing migration" id="create file_meta table" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.986988714Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.987606577Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.988452137Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.988876333Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.989294702Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.990127543Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.991188415Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.991964617Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-05T03:02:27.281 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.992750833Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.993607321Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.994538796Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.995072423Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.996105295Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.997111815Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:26.998095767Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.007925827Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.008875735Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.009653214Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.013056553Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.013389557Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.014203591Z level=info msg="Executing migration" id="create folder table" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.014624852Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-05T03:02:27.282 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.015086393Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.015886684Z level=info msg="Executing migration" id="Update folder title length" 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.01620316Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=migrator t=2024-04-05T03:02:27.017148468Z level=info msg="migrations completed" performed=485 skipped=0 duration=666.533579ms 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=sqlstore t=2024-04-05T03:02:27.017749049Z level=info msg="Created default organization" 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=plugin.loader t=2024-04-05T03:02:27.075374876Z level=info msg="Plugin registered" pluginID=input 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=plugin.loader t=2024-04-05T03:02:27.101972962Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=plugin.loader t=2024-04-05T03:02:27.102002239Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=secrets t=2024-04-05T03:02:27.102050327Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=query_data t=2024-04-05T03:02:27.103755678Z level=info msg="Query Service initialization" 2024-04-05T03:02:27.283 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=live.push_http t=2024-04-05T03:02:27.106245777Z level=info msg="Live Push Gateway initialization" 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[29244]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2038977600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1671162758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[40008]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:27.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2038977600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:02:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:27 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1671162758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:02:27.846 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462934 for osd.6 2024-04-05T03:02:27.847 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:28.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:28 smithi151 ceph-mon[32789]: Deploying daemon node-exporter.a on smithi033 2024-04-05T03:02:28.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=infra.usagestats.collector t=2024-04-05T03:02:27.865580255Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-05T03:02:28.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=provisioning.datasources t=2024-04-05T03:02:27.865989807Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-05T03:02:28.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=provisioning.datasources t=2024-04-05T03:02:27.898417898Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-05T03:02:28.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=provisioning.datasources t=2024-04-05T03:02:27.930359401Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-05T03:02:28.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=provisioning.alerting t=2024-04-05T03:02:27.95242279Z level=info msg="starting to provision alerting" 2024-04-05T03:02:28.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=provisioning.alerting t=2024-04-05T03:02:27.952465961Z level=info msg="finished to provision alerting" 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=ngalert.state.manager t=2024-04-05T03:02:27.952586282Z level=info msg="Warming state cache for startup" 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=grafanaStorageLogger t=2024-04-05T03:02:27.95262571Z level=info msg="storage starting" 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=http.server t=2024-04-05T03:02:27.954397239Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=ngalert.state.manager t=2024-04-05T03:02:27.962910469Z level=info msg="State cache has been initialized" states=0 duration=10.322604ms 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=ticker t=2024-04-05T03:02:27.962982353Z level=info msg=starting first_tick=2024-04-05T03:02:30Z 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=ngalert.multiorg.alertmanager t=2024-04-05T03:02:27.962970271Z level=info msg="starting MultiOrg Alertmanager" 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=sqlstore.transactions t=2024-04-05T03:02:28.01190916Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=sqlstore.transactions t=2024-04-05T03:02:28.05332476Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-05T03:02:28.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:02:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[62176]: logger=sqlstore.transactions t=2024-04-05T03:02:28.063782318Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-05T03:02:28.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:28 smithi033 ceph-mon[29244]: Deploying daemon node-exporter.a on smithi033 2024-04-05T03:02:28.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:28 smithi033 ceph-mon[40008]: Deploying daemon node-exporter.a on smithi033 2024-04-05T03:02:28.481 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:28 smithi033 systemd[1]: Starting Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:02:29.211 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:29 smithi033 ceph-mon[29244]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:29.211 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:29.211 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:29.211 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:29 smithi033 ceph-mon[40008]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:29.212 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:29.212 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:29.239 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149732 for osd.3 2024-04-05T03:02:29.239 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:29 smithi151 ceph-mon[32789]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:29.758 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724566 got 167503724569 for osd.5 2024-04-05T03:02:29.759 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:31.369 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:31 smithi033 ceph-mon[29244]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:31.370 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:31 smithi033 ceph-mon[40008]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:31.461 INFO:teuthology.orchestra.run.smithi033.stdout:55834574895 2024-04-05T03:02:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:31 smithi151 ceph-mon[32789]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:31.906 INFO:teuthology.orchestra.run.smithi033.stdout:236223201298 2024-04-05T03:02:31.980 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:31 smithi033 bash[72273]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-05T03:02:32.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:32 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/727685508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:02:32.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:32 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/147627324' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:02:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:32 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/727685508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:02:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:32 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/147627324' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:02:32.481 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:32 smithi033 bash[72273]: Getting image source signatures 2024-04-05T03:02:32.481 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:32 smithi033 bash[72273]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-05T03:02:32.481 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:32 smithi033 bash[72273]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-05T03:02:32.481 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:32 smithi033 bash[72273]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-05T03:02:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:32 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/727685508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:02:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:32 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/147627324' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:02:32.664 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574895 for osd.1 2024-04-05T03:02:32.664 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:33.129 INFO:teuthology.orchestra.run.smithi033.stdout:77309411369 2024-04-05T03:02:33.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:33 smithi033 ceph-mon[29244]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:33.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:33 smithi033 ceph-mon[40008]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:33.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:33 smithi151 ceph-mon[32789]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:33.881 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201298 for osd.7 2024-04-05T03:02:33.881 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:33.998 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:33 smithi033 bash[72273]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-05T03:02:34.480 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 bash[72273]: Writing manifest to image destination 2024-04-05T03:02:34.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4189266955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:02:34.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4189266955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:02:34.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:34 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4189266955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:02:34.670 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411369 for osd.2 2024-04-05T03:02:34.670 DEBUG:teuthology.parallel:result is None 2024-04-05T03:02:34.670 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-05T03:02:34.671 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:02:34.964 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 podman[72273]: 2024-04-05 03:02:34.556140717 +0000 UTC m=+3.065927276 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T03:02:34.964 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 podman[72273]: 2024-04-05 03:02:34.771404762 +0000 UTC m=+3.281191308 container create 94bec9c472c2c12fca24d82386a4dea7f425bde989ef1efad6166d2f349feaa7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:34.964 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 podman[72273]: 2024-04-05 03:02:34.96515477 +0000 UTC m=+3.474941316 container init 94bec9c472c2c12fca24d82386a4dea7f425bde989ef1efad6166d2f349feaa7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:34.973 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 podman[72273]: 2024-04-05 03:02:34.9679624 +0000 UTC m=+3.477748955 container start 94bec9c472c2c12fca24d82386a4dea7f425bde989ef1efad6166d2f349feaa7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:34.973 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.971Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.971Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.972Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.972Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.972Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.972Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T03:02:34.974 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T03:02:34.975 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T03:02:34.976 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T03:02:34.977 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T03:02:34.977 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T03:02:34.977 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T03:02:34.977 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T03:02:35.171 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:35 smithi033 ceph-mon[29244]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:35 smithi033 ceph-mon[40008]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T03:02:35.231 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T03:02:35.232 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T03:02:35.232 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T03:02:35.232 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[72443]: ts=2024-04-05T03:02:34.973Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T03:02:35.232 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:35 smithi033 bash[72273]: 94bec9c472c2c12fca24d82386a4dea7f425bde989ef1efad6166d2f349feaa7 2024-04-05T03:02:35.232 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:02:35 smithi033 systemd[1]: Started Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:35.476 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:35 smithi151 ceph-mon[32789]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:36.145 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:36 smithi151 bash[62447]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[29244]: Deploying daemon node-exporter.b on smithi151 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.403 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:36 smithi033 ceph-mon[40008]: Deploying daemon node-exporter.b on smithi151 2024-04-05T03:02:36.429 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:36 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.429 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:36 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.429 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:36 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:36.429 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:36 smithi151 ceph-mon[32789]: Deploying daemon node-exporter.b on smithi151 2024-04-05T03:02:36.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:36 smithi151 bash[62447]: Getting image source signatures 2024-04-05T03:02:36.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:36 smithi151 bash[62447]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-05T03:02:36.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:36 smithi151 bash[62447]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-05T03:02:36.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:36 smithi151 bash[62447]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-05T03:02:36.944 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:02:36.946 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:02:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:37 smithi151 ceph-mon[32789]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:37.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:02:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:02:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:02:37.438 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:37 smithi033 ceph-mon[40008]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:37.439 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:37 smithi033 ceph-mon[29244]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:37.753 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":30,"stamp":"2024-04-05T03:02:36.042221+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385492,"kb_used_data":6628,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575724,"statfs":{"total":767960285184,"available":765517541376,"internally_reserved":0,"allocated":6787072,"data_stored":3848939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001950"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035193+0000","last_change":"2024-04-05T03:01:43.659234+0000","last_active":"2024-04-05T03:01:44.035193+0000","last_peered":"2024-04-05T03:01:44.035193+0000","last_clean":"2024-04-05T03:01:44.035193+0000","last_became_active":"2024-04-05T03:01:43.658801+0000","last_became_peered":"2024-04-05T03:01:43.658801+0000","last_unstale":"2024-04-05T03:01:44.035193+0000","last_undegraded":"2024-04-05T03:01:44.035193+0000","last_fullsized":"2024-04-05T03:01:44.035193+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:39:37.401263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035199+0000","last_change":"2024-04-05T03:01:43.656498+0000","last_active":"2024-04-05T03:01:44.035199+0000","last_peered":"2024-04-05T03:01:44.035199+0000","last_clean":"2024-04-05T03:01:44.035199+0000","last_became_active":"2024-04-05T03:01:43.656159+0000","last_became_peered":"2024-04-05T03:01:43.656159+0000","last_unstale":"2024-04-05T03:01:44.035199+0000","last_undegraded":"2024-04-05T03:01:44.035199+0000","last_fullsized":"2024-04-05T03:01:44.035199+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:39.619074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035408+0000","last_change":"2024-04-05T03:01:43.910231+0000","last_active":"2024-04-05T03:01:44.035408+0000","last_peered":"2024-04-05T03:01:44.035408+0000","last_clean":"2024-04-05T03:01:44.035408+0000","last_became_active":"2024-04-05T03:01:43.909886+0000","last_became_peered":"2024-04-05T03:01:43.909886+0000","last_unstale":"2024-04-05T03:01:44.035408+0000","last_undegraded":"2024-04-05T03:01:44.035408+0000","last_fullsized":"2024-04-05T03:01:44.035408+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:22:51.464507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035274+0000","last_change":"2024-04-05T03:01:43.910268+0000","last_active":"2024-04-05T03:01:44.035274+0000","last_peered":"2024-04-05T03:01:44.035274+0000","last_clean":"2024-04-05T03:01:44.035274+0000","last_became_active":"2024-04-05T03:01:43.909991+0000","last_became_peered":"2024-04-05T03:01:43.909991+0000","last_unstale":"2024-04-05T03:01:44.035274+0000","last_undegraded":"2024-04-05T03:01:44.035274+0000","last_fullsized":"2024-04-05T03:01:44.035274+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:12.008125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T03:01:46.093596+0000","last_change":"2024-04-05T03:01:14.837104+0000","last_active":"2024-04-05T03:01:46.093596+0000","last_peered":"2024-04-05T03:01:46.093596+0000","last_clean":"2024-04-05T03:01:46.093596+0000","last_became_active":"2024-04-05T03:01:14.836690+0000","last_became_peered":"2024-04-05T03:01:14.836690+0000","last_unstale":"2024-04-05T03:01:46.093596+0000","last_undegraded":"2024-04-05T03:01:46.093596+0000","last_fullsized":"2024-04-05T03:01:46.093596+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:48:28.544381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.025936+0000","last_change":"2024-04-05T03:01:43.659091+0000","last_active":"2024-04-05T03:01:44.025936+0000","last_peered":"2024-04-05T03:01:44.025936+0000","last_clean":"2024-04-05T03:01:44.025936+0000","last_became_active":"2024-04-05T03:01:43.658783+0000","last_became_peered":"2024-04-05T03:01:43.658783+0000","last_unstale":"2024-04-05T03:01:44.025936+0000","last_undegraded":"2024-04-05T03:01:44.025936+0000","last_fullsized":"2024-04-05T03:01:44.025936+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:34:20.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.028272+0000","last_change":"2024-04-05T03:01:43.910452+0000","last_active":"2024-04-05T03:01:44.028272+0000","last_peered":"2024-04-05T03:01:44.028272+0000","last_clean":"2024-04-05T03:01:44.028272+0000","last_became_active":"2024-04-05T03:01:43.909876+0000","last_became_peered":"2024-04-05T03:01:43.909876+0000","last_unstale":"2024-04-05T03:01:44.028272+0000","last_undegraded":"2024-04-05T03:01:44.028272+0000","last_fullsized":"2024-04-05T03:01:44.028272+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:07:36.880582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.025924+0000","last_change":"2024-04-05T03:01:43.656346+0000","last_active":"2024-04-05T03:01:44.025924+0000","last_peered":"2024-04-05T03:01:44.025924+0000","last_clean":"2024-04-05T03:01:44.025924+0000","last_became_active":"2024-04-05T03:01:43.656045+0000","last_became_peered":"2024-04-05T03:01:43.656045+0000","last_unstale":"2024-04-05T03:01:44.025924+0000","last_undegraded":"2024-04-05T03:01:44.025924+0000","last_fullsized":"2024-04-05T03:01:44.025924+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:40:55.612040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035200+0000","last_change":"2024-04-05T03:01:43.910340+0000","last_active":"2024-04-05T03:01:44.035200+0000","last_peered":"2024-04-05T03:01:44.035200+0000","last_clean":"2024-04-05T03:01:44.035200+0000","last_became_active":"2024-04-05T03:01:43.909987+0000","last_became_peered":"2024-04-05T03:01:43.909987+0000","last_unstale":"2024-04-05T03:01:44.035200+0000","last_undegraded":"2024-04-05T03:01:44.035200+0000","last_fullsized":"2024-04-05T03:01:44.035200+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:42.918552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":850161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.502}]},{"osd":1,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.68500000000000005}]},{"osd":2,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.628},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.55100000000000005}]},{"osd":3,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.71199999999999997}]},{"osd":4,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59799999999999998}]},{"osd":5,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.76100000000000001}]},{"osd":6,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.69899999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298544,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446608,"statfs":{"total":95995035648,"available":95689326592,"internally_reserved":0,"allocated":1216512,"data_stored":849917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.54300000000000004}]},{"osd":1,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.46500000000000002}]},{"osd":2,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.47599999999999998}]},{"osd":3,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.49399999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.435},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.60299999999999998}]},{"osd":5,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.67200000000000004}]},{"osd":7,"last update":"Fri Apr 5 03:02:12 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.70399999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297964,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447188,"statfs":{"total":95995035648,"available":95689920512,"internally_reserved":0,"allocated":622592,"data_stored":259528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:01:50 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.33100000000000002,"15min":0.309},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.56999999999999995,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.36899999999999999,"15min":0.34100000000000003},"min":{"1min":0.20999999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.54900000000000004}]},{"osd":2,"last update":"Fri Apr 5 03:02:18 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40200000000000002,"15min":0.39200000000000002},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60399999999999998,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.434,"5min":0.38700000000000001,"15min":0.36899999999999999},"min":{"1min":0.24199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.7030000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Fri Apr 5 03:01:45 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42699999999999999,"15min":0.42399999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.46500000000000002,"15min":0.46300000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.48899999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:02:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.63500000000000001,"15min":0.64000000000000001},"min":{"1min":0.41999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.42099999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83799999999999997,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.67700000000000005}]},{"osd":5,"last update":"Fri Apr 5 03:02:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.40600000000000003,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.745,"5min":0.879,"15min":0.879},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.64200000000000002,"15min":0.64400000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.7709999999999999,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":0.626}]},{"osd":6,"last update":"Fri Apr 5 03:01:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.64400000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.73999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":850161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 03:01:54 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34599999999999997,"15min":0.32100000000000001},"min":{"1min":0.27300000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.439,"5min":0.35999999999999999,"15min":0.33600000000000002},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Apr 5 03:02:21 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.38900000000000001},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.40899999999999997,"15min":0.39600000000000002},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.31}]},{"osd":3,"last update":"Fri Apr 5 03:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45700000000000002,"15min":0.45500000000000002},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.373}]},{"osd":4,"last update":"Fri Apr 5 03:01:59 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.57999999999999996,"15min":0.57599999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.59399999999999997,"15min":0.58799999999999997},"min":{"1min":0.46300000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.51400000000000001}]},{"osd":5,"last update":"Fri Apr 5 03:02:24 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61599999999999999,"15min":0.62},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.78100000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.65700000000000003,"15min":0.66100000000000003},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.89800000000000002,"5min":1.018,"15min":1.018},"last":0.53800000000000003}]},{"osd":6,"last update":"Fri Apr 5 03:01:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.46999999999999997}]},{"osd":7,"last update":"Fri Apr 5 03:02:15 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.56399999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:18 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.38400000000000001,"15min":0.38700000000000001},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.44900000000000001,"5min":0.623,"15min":0.623},"last":0.433},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.39200000000000002,"15min":0.39500000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.495,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51900000000000002}]},{"osd":1,"last update":"Fri Apr 5 03:02:18 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.40000000000000002,"15min":0.40699999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.46500000000000002,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.379,"5min":0.42599999999999999,"15min":0.43099999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60999999999999999,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.45300000000000001}]},{"osd":3,"last update":"Fri Apr 5 03:01:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46899999999999997,"15min":0.46700000000000003},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.505},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.45900000000000002,"15min":0.46100000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.46899999999999997}]},{"osd":4,"last update":"Fri Apr 5 03:01:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.45400000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.72899999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77600000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.78100000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:02:29 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56899999999999995,"15min":0.57199999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.83299999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.56899999999999995,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71099999999999997,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.70799999999999996}]},{"osd":6,"last update":"Fri Apr 5 03:01:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.73599999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.84199999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:01:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.39100000000000001,"15min":0.38900000000000001},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.56399999999999995,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.443,"15min":0.44600000000000001},"min":{"1min":0.29899999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.55500000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.35499999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:01:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.44900000000000001,"15min":0.44800000000000001},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.629,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.45400000000000001,"15min":0.45800000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.61699999999999999,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.253}]},{"osd":2,"last update":"Fri Apr 5 03:01:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.499,"15min":0.501},"min":{"1min":0.311,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.67000000000000004,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.38},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.36699999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:02:03 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.65500000000000003,"15min":0.66100000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85199999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.65400000000000003,"15min":0.66100000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90500000000000003,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.61799999999999999}]},{"osd":5,"last update":"Fri Apr 5 03:02:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83499999999999996,"5min":1.002,"15min":1.002},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.63500000000000001,"15min":0.64300000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.79700000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.53500000000000003}]},{"osd":6,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.627}]},{"osd":7,"last update":"Fri Apr 5 03:02:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.58599999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.55500000000000005,"15min":0.56399999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.623,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.55900000000000005,"15min":0.56399999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74299999999999999,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.46000000000000002}]},{"osd":1,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.55100000000000005,"15min":0.55600000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.44},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.80900000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.58799999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53100000000000003,"15min":0.53500000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58899999999999997,"15min":0.59599999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.36199999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.57199999999999995,"15min":0.57899999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.497}]},{"osd":5,"last update":"Fri Apr 5 03:02:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.40600000000000003,"15min":0.39800000000000002},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.439,"15min":0.42999999999999999},"min":{"1min":0.31900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.56299999999999994}]},{"osd":6,"last update":"Fri Apr 5 03:01:49 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.623},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.61099999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:02:16 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.54600000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.52700000000000002,"15min":0.52100000000000002},"min":{"1min":0.42399999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.56899999999999995}]},{"osd":1,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.41499999999999998,"5min":0.376,"15min":0.376},"max":{"1min":0.72099999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.55700000000000005,"15min":0.55400000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.50800000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56000000000000005,"15min":0.55900000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.54000000000000004,"15min":0.53300000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80600000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.54400000000000004}]},{"osd":3,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.40000000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.71899999999999997,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.496},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56200000000000006,"15min":0.55600000000000005},"min":{"1min":0.40100000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.55900000000000005}]},{"osd":4,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.39200000000000002,"15min":0.376},"min":{"1min":0.33100000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.41099999999999998,"15min":0.39500000000000002},"min":{"1min":0.315,"5min":0.25,"15min":0.25},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.434}]},{"osd":6,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.40300000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:02:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.45300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:02:37.755 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:02:38.021 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:38.027 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:37 smithi151 bash[62447]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-05T03:02:38.028 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:37 smithi151 bash[62447]: Writing manifest to image destination 2024-04-05T03:02:38.375 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-mon[32789]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:02:38.375 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 podman[62447]: 2024-04-05 03:02:38.257665944 +0000 UTC m=+2.305145909 container create 958473fd6ad3e1775df94cc8e304fe295b80304a55bc00964c0ea1ad62a50afa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:02:38.375 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 podman[62447]: 2024-04-05 03:02:38.17546959 +0000 UTC m=+2.222949574 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T03:02:38.451 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:38 smithi033 ceph-mon[40008]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:02:38.451 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:38 smithi033 ceph-mon[29244]: from='client.24524 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:02:38.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 podman[62447]: 2024-04-05 03:02:38.43455125 +0000 UTC m=+2.482031221 container init 958473fd6ad3e1775df94cc8e304fe295b80304a55bc00964c0ea1ad62a50afa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:02:38.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 podman[62447]: 2024-04-05 03:02:38.437181365 +0000 UTC m=+2.484661330 container start 958473fd6ad3e1775df94cc8e304fe295b80304a55bc00964c0ea1ad62a50afa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:02:38.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.440Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.440Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.441Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.441Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.441Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.441Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T03:02:38.779 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T03:02:38.780 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T03:02:38.781 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T03:02:38.782 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.442Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T03:02:38.783 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.443Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T03:02:38.783 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[62496]: ts=2024-04-05T03:02:38.443Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T03:02:38.783 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 bash[62447]: 958473fd6ad3e1775df94cc8e304fe295b80304a55bc00964c0ea1ad62a50afa 2024-04-05T03:02:38.783 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:02:38 smithi151 systemd[1]: Started Ceph node-exporter.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:39.386 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:39.386 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.386 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.386 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:39.387 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:39 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:39.418 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:39 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:39.817 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:02:39.819 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:02:40.471 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":31,"stamp":"2024-04-05T03:02:38.042695+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385492,"kb_used_data":6628,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575724,"statfs":{"total":767960285184,"available":765517541376,"internally_reserved":0,"allocated":6787072,"data_stored":3848939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002159"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035193+0000","last_change":"2024-04-05T03:01:43.659234+0000","last_active":"2024-04-05T03:01:44.035193+0000","last_peered":"2024-04-05T03:01:44.035193+0000","last_clean":"2024-04-05T03:01:44.035193+0000","last_became_active":"2024-04-05T03:01:43.658801+0000","last_became_peered":"2024-04-05T03:01:43.658801+0000","last_unstale":"2024-04-05T03:01:44.035193+0000","last_undegraded":"2024-04-05T03:01:44.035193+0000","last_fullsized":"2024-04-05T03:01:44.035193+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:39:37.401263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035199+0000","last_change":"2024-04-05T03:01:43.656498+0000","last_active":"2024-04-05T03:01:44.035199+0000","last_peered":"2024-04-05T03:01:44.035199+0000","last_clean":"2024-04-05T03:01:44.035199+0000","last_became_active":"2024-04-05T03:01:43.656159+0000","last_became_peered":"2024-04-05T03:01:43.656159+0000","last_unstale":"2024-04-05T03:01:44.035199+0000","last_undegraded":"2024-04-05T03:01:44.035199+0000","last_fullsized":"2024-04-05T03:01:44.035199+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:39.619074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035408+0000","last_change":"2024-04-05T03:01:43.910231+0000","last_active":"2024-04-05T03:01:44.035408+0000","last_peered":"2024-04-05T03:01:44.035408+0000","last_clean":"2024-04-05T03:01:44.035408+0000","last_became_active":"2024-04-05T03:01:43.909886+0000","last_became_peered":"2024-04-05T03:01:43.909886+0000","last_unstale":"2024-04-05T03:01:44.035408+0000","last_undegraded":"2024-04-05T03:01:44.035408+0000","last_fullsized":"2024-04-05T03:01:44.035408+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:22:51.464507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035274+0000","last_change":"2024-04-05T03:01:43.910268+0000","last_active":"2024-04-05T03:01:44.035274+0000","last_peered":"2024-04-05T03:01:44.035274+0000","last_clean":"2024-04-05T03:01:44.035274+0000","last_became_active":"2024-04-05T03:01:43.909991+0000","last_became_peered":"2024-04-05T03:01:43.909991+0000","last_unstale":"2024-04-05T03:01:44.035274+0000","last_undegraded":"2024-04-05T03:01:44.035274+0000","last_fullsized":"2024-04-05T03:01:44.035274+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:12.008125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T03:01:46.093596+0000","last_change":"2024-04-05T03:01:14.837104+0000","last_active":"2024-04-05T03:01:46.093596+0000","last_peered":"2024-04-05T03:01:46.093596+0000","last_clean":"2024-04-05T03:01:46.093596+0000","last_became_active":"2024-04-05T03:01:14.836690+0000","last_became_peered":"2024-04-05T03:01:14.836690+0000","last_unstale":"2024-04-05T03:01:46.093596+0000","last_undegraded":"2024-04-05T03:01:46.093596+0000","last_fullsized":"2024-04-05T03:01:46.093596+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:48:28.544381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.025936+0000","last_change":"2024-04-05T03:01:43.659091+0000","last_active":"2024-04-05T03:01:44.025936+0000","last_peered":"2024-04-05T03:01:44.025936+0000","last_clean":"2024-04-05T03:01:44.025936+0000","last_became_active":"2024-04-05T03:01:43.658783+0000","last_became_peered":"2024-04-05T03:01:43.658783+0000","last_unstale":"2024-04-05T03:01:44.025936+0000","last_undegraded":"2024-04-05T03:01:44.025936+0000","last_fullsized":"2024-04-05T03:01:44.025936+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:34:20.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.028272+0000","last_change":"2024-04-05T03:01:43.910452+0000","last_active":"2024-04-05T03:01:44.028272+0000","last_peered":"2024-04-05T03:01:44.028272+0000","last_clean":"2024-04-05T03:01:44.028272+0000","last_became_active":"2024-04-05T03:01:43.909876+0000","last_became_peered":"2024-04-05T03:01:43.909876+0000","last_unstale":"2024-04-05T03:01:44.028272+0000","last_undegraded":"2024-04-05T03:01:44.028272+0000","last_fullsized":"2024-04-05T03:01:44.028272+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:07:36.880582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.025924+0000","last_change":"2024-04-05T03:01:43.656346+0000","last_active":"2024-04-05T03:01:44.025924+0000","last_peered":"2024-04-05T03:01:44.025924+0000","last_clean":"2024-04-05T03:01:44.025924+0000","last_became_active":"2024-04-05T03:01:43.656045+0000","last_became_peered":"2024-04-05T03:01:43.656045+0000","last_unstale":"2024-04-05T03:01:44.025924+0000","last_undegraded":"2024-04-05T03:01:44.025924+0000","last_fullsized":"2024-04-05T03:01:44.025924+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:40:55.612040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-05T03:01:44.035200+0000","last_change":"2024-04-05T03:01:43.910340+0000","last_active":"2024-04-05T03:01:44.035200+0000","last_peered":"2024-04-05T03:01:44.035200+0000","last_clean":"2024-04-05T03:01:44.035200+0000","last_became_active":"2024-04-05T03:01:43.909987+0000","last_became_peered":"2024-04-05T03:01:43.909987+0000","last_unstale":"2024-04-05T03:01:44.035200+0000","last_undegraded":"2024-04-05T03:01:44.035200+0000","last_fullsized":"2024-04-05T03:01:44.035200+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:42.918552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":850161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.502}]},{"osd":1,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.68500000000000005}]},{"osd":2,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.628},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.55100000000000005}]},{"osd":3,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.71199999999999997}]},{"osd":4,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59799999999999998}]},{"osd":5,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.76100000000000001}]},{"osd":6,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.69899999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298544,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446608,"statfs":{"total":95995035648,"available":95689326592,"internally_reserved":0,"allocated":1216512,"data_stored":849917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.54300000000000004}]},{"osd":1,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.46500000000000002}]},{"osd":2,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.47599999999999998}]},{"osd":3,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.49399999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.435},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.60299999999999998}]},{"osd":5,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.67200000000000004}]},{"osd":7,"last update":"Fri Apr 5 03:02:12 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.70399999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297964,"kb_used_data":608,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447188,"statfs":{"total":95995035648,"available":95689920512,"internally_reserved":0,"allocated":622592,"data_stored":259528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:01:50 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.33100000000000002,"15min":0.309},"min":{"1min":0.253,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.56999999999999995,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.36899999999999999,"15min":0.34100000000000003},"min":{"1min":0.20999999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.54900000000000004}]},{"osd":2,"last update":"Fri Apr 5 03:02:18 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40200000000000002,"15min":0.39200000000000002},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60399999999999998,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.434,"5min":0.38700000000000001,"15min":0.36899999999999999},"min":{"1min":0.24199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.7030000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Fri Apr 5 03:01:45 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42699999999999999,"15min":0.42399999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.46500000000000002,"15min":0.46300000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.48899999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:02:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.63500000000000001,"15min":0.64000000000000001},"min":{"1min":0.41999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.42099999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83799999999999997,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.67700000000000005}]},{"osd":5,"last update":"Fri Apr 5 03:02:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.40600000000000003,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.745,"5min":0.879,"15min":0.879},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.64200000000000002,"15min":0.64400000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.7709999999999999,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":0.626}]},{"osd":6,"last update":"Fri Apr 5 03:01:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.64400000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.73999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298548,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446604,"statfs":{"total":95995035648,"available":95689322496,"internally_reserved":0,"allocated":1220608,"data_stored":850161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 03:01:54 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.34599999999999997,"15min":0.32100000000000001},"min":{"1min":0.27300000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.439,"5min":0.35999999999999999,"15min":0.33600000000000002},"min":{"1min":0.25900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Apr 5 03:02:21 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.38900000000000001},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.40899999999999997,"15min":0.39600000000000002},"min":{"1min":0.29699999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.31}]},{"osd":3,"last update":"Fri Apr 5 03:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45700000000000002,"15min":0.45500000000000002},"min":{"1min":0.34000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.373}]},{"osd":4,"last update":"Fri Apr 5 03:01:59 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.57999999999999996,"15min":0.57599999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.59399999999999997,"15min":0.58799999999999997},"min":{"1min":0.46300000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.51400000000000001}]},{"osd":5,"last update":"Fri Apr 5 03:02:24 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61599999999999999,"15min":0.62},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.78100000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.65700000000000003,"15min":0.66100000000000003},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.89800000000000002,"5min":1.018,"15min":1.018},"last":0.53800000000000003}]},{"osd":6,"last update":"Fri Apr 5 03:01:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.46999999999999997}]},{"osd":7,"last update":"Fri Apr 5 03:02:15 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.56399999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:18 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.38400000000000001,"15min":0.38700000000000001},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.44900000000000001,"5min":0.623,"15min":0.623},"last":0.433},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.39200000000000002,"15min":0.39500000000000002},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.495,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51900000000000002}]},{"osd":1,"last update":"Fri Apr 5 03:02:18 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.40000000000000002,"15min":0.40699999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.46500000000000002,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.379,"5min":0.42599999999999999,"15min":0.43099999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.60999999999999999,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.45300000000000001}]},{"osd":3,"last update":"Fri Apr 5 03:01:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46899999999999997,"15min":0.46700000000000003},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.505},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.45900000000000002,"15min":0.46100000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.46899999999999997}]},{"osd":4,"last update":"Fri Apr 5 03:01:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.45400000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.72899999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77600000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.78100000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:02:29 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.56899999999999995,"15min":0.57199999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.83299999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.56899999999999995,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71099999999999997,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.70799999999999996}]},{"osd":6,"last update":"Fri Apr 5 03:01:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.73599999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:02:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.84199999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:01:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.39100000000000001,"15min":0.38900000000000001},"min":{"1min":0.318,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.56399999999999995,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.443,"15min":0.44600000000000001},"min":{"1min":0.29899999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.55500000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.35499999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:01:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.44900000000000001,"15min":0.44800000000000001},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.629,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.45400000000000001,"15min":0.45800000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.61699999999999999,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.253}]},{"osd":2,"last update":"Fri Apr 5 03:01:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.499,"15min":0.501},"min":{"1min":0.311,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.67000000000000004,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.38},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.36699999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:02:03 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.65500000000000003,"15min":0.66100000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.85199999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.65400000000000003,"15min":0.66100000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.90500000000000003,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.61799999999999999}]},{"osd":5,"last update":"Fri Apr 5 03:02:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.60199999999999998,"15min":0.60799999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83499999999999996,"5min":1.002,"15min":1.002},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.63500000000000001,"15min":0.64300000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.79700000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.53500000000000003}]},{"osd":6,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.627}]},{"osd":7,"last update":"Fri Apr 5 03:02:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.58599999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.55500000000000005,"15min":0.56399999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.623,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.55900000000000005,"15min":0.56399999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74299999999999999,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.46000000000000002}]},{"osd":1,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.55100000000000005,"15min":0.55600000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.44},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.80900000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.58799999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.53100000000000003,"15min":0.53500000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58899999999999997,"15min":0.59599999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.36199999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:02:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.57199999999999995,"15min":0.57899999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.497}]},{"osd":5,"last update":"Fri Apr 5 03:02:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.40600000000000003,"15min":0.39800000000000002},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.439,"15min":0.42999999999999999},"min":{"1min":0.31900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.56299999999999994}]},{"osd":6,"last update":"Fri Apr 5 03:01:49 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.623},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.61099999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:02:16 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.54600000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":612,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":626688,"data_stored":259793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.52700000000000002,"15min":0.52100000000000002},"min":{"1min":0.42399999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.56899999999999995}]},{"osd":1,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.41499999999999998,"5min":0.376,"15min":0.376},"max":{"1min":0.72099999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.55700000000000005,"15min":0.55400000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.50800000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56000000000000005,"15min":0.55900000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.54000000000000004,"15min":0.53300000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80600000000000005,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.54400000000000004}]},{"osd":3,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.40000000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.71899999999999997,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.496},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56200000000000006,"15min":0.55600000000000005},"min":{"1min":0.40100000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.55900000000000005}]},{"osd":4,"last update":"Fri Apr 5 03:02:25 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.39200000000000002,"15min":0.376},"min":{"1min":0.33100000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.41099999999999998,"15min":0.39500000000000002},"min":{"1min":0.315,"5min":0.25,"15min":0.25},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.434}]},{"osd":6,"last update":"Fri Apr 5 03:01:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.40300000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:02:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.45300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:02:40.473 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-05T03:02:40.474 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-05T03:02:40.474 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-05T03:02:40.474 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:02:40.821 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:41.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:41 smithi151 ceph-mon[32789]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:02:41.444 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:41 smithi151 ceph-mon[32789]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:41.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:41 smithi033 ceph-mon[40008]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:02:41.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:41 smithi033 ceph-mon[40008]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:41.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:41 smithi033 ceph-mon[29244]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:02:41.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:41 smithi033 ceph-mon[29244]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:42.823 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:02:42.823 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-05T03:02:43.140 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:43.140 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:43.140 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[40008]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:43.140 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4086599169' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:02:43.141 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:43.141 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:43.141 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[29244]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:43.141 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4086599169' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:02:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:43 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:43 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:43 smithi151 ceph-mon[32789]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4086599169' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:02:43.899 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-05T03:02:43.900 INFO:tasks.cephadm:Setup complete, yielding 2024-04-05T03:02:43.900 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-05T03:02:43.910 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi033.front.sepia.ceph.com 2024-04-05T03:02:43.911 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- bash -c 'ceph fs volume create foo' 2024-04-05T03:02:44.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:44.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:44.649 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[40008]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[29244]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:02:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:45 smithi151 ceph-mon[32789]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:02:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:02:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:46.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:46 smithi033 ceph-mon[40008]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T03:02:46.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:46 smithi033 ceph-mon[40008]: Reconfiguring daemon alertmanager.a on smithi033 2024-04-05T03:02:46.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:46 smithi033 ceph-mon[29244]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T03:02:46.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:46 smithi033 ceph-mon[29244]: Reconfiguring daemon alertmanager.a on smithi033 2024-04-05T03:02:46.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:46 smithi151 ceph-mon[32789]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T03:02:46.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:46 smithi151 ceph-mon[32789]: Reconfiguring daemon alertmanager.a on smithi033 2024-04-05T03:02:47.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:02:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:02:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:02:47.570 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:47 smithi033 ceph-mon[40008]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:47.571 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:47 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T03:02:47.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:47 smithi033 ceph-mon[29244]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:47.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:47 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T03:02:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:47 smithi151 ceph-mon[32789]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:47 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T03:02:48.653 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[40008]: from='client.24542 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:02:48.653 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-05T03:02:48.653 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[40008]: osdmap e64: 8 total, 8 up, 8 in 2024-04-05T03:02:48.653 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T03:02:48.653 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[40008]: pgmap v37: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:48.653 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[29244]: from='client.24542 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:02:48.653 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-05T03:02:48.654 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[29244]: osdmap e64: 8 total, 8 up, 8 in 2024-04-05T03:02:48.654 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T03:02:48.654 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-mon[29244]: pgmap v37: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:48.654 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:48 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a[29240]: 2024-04-05T03:02:48.449+0000 7f110a3cd700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T03:02:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:48 smithi151 ceph-mon[32789]: from='client.24542 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:02:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:48 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-05T03:02:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:48 smithi151 ceph-mon[32789]: osdmap e64: 8 total, 8 up, 8 in 2024-04-05T03:02:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:48 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T03:02:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:48 smithi151 ceph-mon[32789]: pgmap v37: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:48.944 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:48 smithi033 systemd[1]: Stopping Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:02:49.341 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-05T03:02:49.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[70569]: ts=2024-04-05T03:02:49.461Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:02:49.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:49 smithi033 podman[73761]: 2024-04-05 03:02:49.502327429 +0000 UTC m=+0.147583897 container died a83da33c8a66f1230c6361a5f561ba2a7935a4e40025c0fdddb1ba38b1d50f21 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:02:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-05T03:02:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: osdmap e65: 8 total, 8 up, 8 in 2024-04-05T03:02:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T03:02:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T03:02:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-05T03:02:49.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: osdmap e66: 8 total, 8 up, 8 in 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: fsmap foo:0 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: Saving service mds.foo spec with placement count:2 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: osdmap e65: 8 total, 8 up, 8 in 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: osdmap e66: 8 total, 8 up, 8 in 2024-04-05T03:02:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-05T03:02:49.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: fsmap foo:0 2024-04-05T03:02:49.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: Saving service mds.foo spec with placement count:2 2024-04-05T03:02:49.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:49 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:02:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-05T03:02:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: osdmap e65: 8 total, 8 up, 8 in 2024-04-05T03:02:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T03:02:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T03:02:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-05T03:02:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: osdmap e66: 8 total, 8 up, 8 in 2024-04-05T03:02:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-05T03:02:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: fsmap foo:0 2024-04-05T03:02:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: Saving service mds.foo spec with placement count:2 2024-04-05T03:02:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:49 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:49.957 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:50.137 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:49 smithi033 podman[73761]: 2024-04-05 03:02:49.870284815 +0000 UTC m=+0.515541287 container cleanup a83da33c8a66f1230c6361a5f561ba2a7935a4e40025c0fdddb1ba38b1d50f21 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:50.137 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:49 smithi033 bash[73761]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-05T03:02:50.413 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:50 smithi033 podman[73786]: 2024-04-05 03:02:50.13813997 +0000 UTC m=+0.636908533 container remove a83da33c8a66f1230c6361a5f561ba2a7935a4e40025c0fdddb1ba38b1d50f21 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:50.413 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:50 smithi033 podman[73786]: 2024-04-05 03:02:50.222206587 +0000 UTC m=+0.720975105 volume remove 282a33892dc5b01413b42a7441561962fc432839c06bad3c162abe8282faceff 2024-04-05T03:02:50.699 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:50 smithi033 ceph-mon[29244]: osdmap e67: 8 total, 8 up, 8 in 2024-04-05T03:02:50.699 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:50 smithi033 ceph-mon[29244]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:50.699 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:50 smithi033 ceph-mon[40008]: osdmap e67: 8 total, 8 up, 8 in 2024-04-05T03:02:50.699 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:50 smithi033 ceph-mon[40008]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:50 smithi151 ceph-mon[32789]: osdmap e67: 8 total, 8 up, 8 in 2024-04-05T03:02:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:50 smithi151 ceph-mon[32789]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:51.684 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:51 smithi033 ceph-mon[29244]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:02:51.684 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:51 smithi033 ceph-mon[29244]: osdmap e68: 8 total, 8 up, 8 in 2024-04-05T03:02:51.685 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:51 smithi033 ceph-mon[40008]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:02:51.685 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:51 smithi033 ceph-mon[40008]: osdmap e68: 8 total, 8 up, 8 in 2024-04-05T03:02:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:51 smithi151 ceph-mon[32789]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:02:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:51 smithi151 ceph-mon[32789]: osdmap e68: 8 total, 8 up, 8 in 2024-04-05T03:02:51.980 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:51 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-05T03:02:51.980 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:51 smithi033 systemd[1]: Stopped Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:51.981 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:51 smithi033 systemd[1]: Starting Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:02:52.357 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:52 smithi033 podman[74001]: 2024-04-05 03:02:52.10026756 +0000 UTC m=+0.112246519 volume create 40e20bd7e14ef91247318ed0ac0475fafef9983bd64aaaa6c82b283f03606deb 2024-04-05T03:02:52.357 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:52 smithi033 podman[74001]: 2024-04-05 03:02:52.005713825 +0000 UTC m=+0.017692789 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-05T03:02:52.357 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:52 smithi033 podman[74001]: 2024-04-05 03:02:52.20927851 +0000 UTC m=+0.221257469 container create fc398a28fe815d7bc08205e8aabf1a83913d654cb1c637bef3116eabca82f4c4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:52.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:52 smithi033 ceph-mon[29244]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:52.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:52 smithi033 ceph-mon[40008]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:52.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:52 smithi151 ceph-mon[32789]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:53.086 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-05T03:02:53.230 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 podman[74001]: 2024-04-05 03:02:53.199604817 +0000 UTC m=+1.211583778 container init fc398a28fe815d7bc08205e8aabf1a83913d654cb1c637bef3116eabca82f4c4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:53.230 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 podman[74001]: 2024-04-05 03:02:53.202605204 +0000 UTC m=+1.214584165 container start fc398a28fe815d7bc08205e8aabf1a83913d654cb1c637bef3116eabca82f4c4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:53.231 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.219Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-05T03:02:53.231 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.219Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-05T03:02:53.231 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.220Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.33 port=9094 2024-04-05T03:02:53.231 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.221Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-05T03:02:53.386 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:53.509 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.248Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T03:02:53.509 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.249Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T03:02:53.509 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.252Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-05T03:02:53.509 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:53.252Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-05T03:02:53.509 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 bash[74001]: fc398a28fe815d7bc08205e8aabf1a83913d654cb1c637bef3116eabca82f4c4 2024-04-05T03:02:53.509 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:53 smithi033 systemd[1]: Started Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:54.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:54 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:54.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:54 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:54.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:54 smithi151 ceph-mon[32789]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:02:54.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:54 smithi151 ceph-mon[32789]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:02:54.456 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:54 smithi151 ceph-mon[32789]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:54.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:54.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[40008]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[40008]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[40008]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[29244]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[29244]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:02:54.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:54 smithi033 ceph-mon[29244]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:55.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:02:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:02:55.222Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000953584s 2024-04-05T03:02:55.856 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-05T03:02:55.867 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-05T03:02:55.867 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-05T03:02:55.867 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-05T03:02:56.132 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:57.022 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:56 smithi151 systemd[1]: Stopping Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:02:57.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-mon[32789]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:57.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:02:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:02:57.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:02:57.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T03:02:57.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.074Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.075Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.075Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[60137]: ts=2024-04-05T03:02:57.075Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T03:02:57.279 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 podman[63269]: 2024-04-05 03:02:57.084843997 +0000 UTC m=+0.127790580 container died 363364dc408900bf698c5386ca275474eac0f6a4ff2701fb2ab8b3ebf51e5805 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:57.361 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:57 smithi033 ceph-mon[29244]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:57.361 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:57 smithi033 ceph-mon[40008]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:57.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 podman[63269]: 2024-04-05 03:02:57.633415164 +0000 UTC m=+0.676361757 container cleanup 363364dc408900bf698c5386ca275474eac0f6a4ff2701fb2ab8b3ebf51e5805 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:57.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 bash[63269]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T03:02:57.920 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:02:58.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2614133328' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:02:58.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2614133328' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:02:58.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2614133328' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:02:58.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:57 smithi151 podman[63283]: 2024-04-05 03:02:57.934871476 +0000 UTC m=+0.853173366 container remove 363364dc408900bf698c5386ca275474eac0f6a4ff2701fb2ab8b3ebf51e5805 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:58.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-05T03:02:58.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 systemd[1]: Stopped Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:58.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service: Consumed 1.021s CPU time. 2024-04-05T03:02:58.278 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 systemd[1]: Starting Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:02:58.562 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":20,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":2530405471},{"type":"v1","addr":"172.21.15.33:6801","nonce":2530405471}]},"active_addr":"172.21.15.33:6801/2530405471","active_change":"2024-04-05T03:01:44.019851+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.33:8443/","prometheus":"http://172.21.15.33:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":62,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":2236066571}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3439747338}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":1920159030}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3813037768}]}]} 2024-04-05T03:02:58.567 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-05T03:02:58.567 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-05T03:02:58.567 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:02:58.579 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 podman[63381]: 2024-04-05 03:02:58.219949529 +0000 UTC m=+0.018952863 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T03:02:58.579 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 podman[63381]: 2024-04-05 03:02:58.327561644 +0000 UTC m=+0.126564971 container create c19cfcd8e6480c422454be1146b535fd0dab98f583ac7c32a47a20fb13dc9aa8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:58.579 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 podman[63381]: 2024-04-05 03:02:58.554485759 +0000 UTC m=+0.353489082 container init c19cfcd8e6480c422454be1146b535fd0dab98f583ac7c32a47a20fb13dc9aa8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:58.579 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 podman[63381]: 2024-04-05 03:02:58.557476258 +0000 UTC m=+0.356479576 container start c19cfcd8e6480c422454be1146b535fd0dab98f583ac7c32a47a20fb13dc9aa8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:02:58.815 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STOPPING 2024-04-05T03:02:58.815 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:02:58.815 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STOPPED 2024-04-05T03:02:58.816 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STARTING 2024-04-05T03:02:58.816 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Serving on http://:::9283 2024-04-05T03:02:58.816 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STARTED 2024-04-05T03:02:58.853 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.580Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.580Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.580Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi151 (none))" 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.580Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.580Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.582Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.582Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.583Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.583Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.585Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T03:02:59.029 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.585Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.364µs 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.585Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.585Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.586Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.586Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.121µs wal_replay_duration=831.422µs wbl_replay_duration=94ns total_replay_duration=955.172µs 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.586Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.586Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.586Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.600Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.470052ms db_storage=725ns remote_storage=957ns web_handler=392ns query_engine=486ns scrape=376.892µs scrape_sd=21.839µs notify=25.084µs notify_sd=6.354µs rules=12.846692ms tracing=5.398µs 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.600Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:02:58.600Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 bash[63381]: c19cfcd8e6480c422454be1146b535fd0dab98f583ac7c32a47a20fb13dc9aa8 2024-04-05T03:02:59.030 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:02:58 smithi151 systemd[1]: Started Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:02:59.086 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STOPPING 2024-04-05T03:02:59.086 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:02:59.086 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STOPPED 2024-04-05T03:02:59.086 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STARTING 2024-04-05T03:02:59.086 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Serving on http://:::9283 2024-04-05T03:02:59.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STARTED 2024-04-05T03:02:59.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STOPPING 2024-04-05T03:02:59.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:02:59.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STOPPED 2024-04-05T03:02:59.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:58 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:58] ENGINE Bus STARTING 2024-04-05T03:02:59.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:59] ENGINE Serving on http://:::9283 2024-04-05T03:02:59.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:02:59] ENGINE Bus STARTED 2024-04-05T03:02:59.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T03:02:59.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:02:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.483 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:02:59.484 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:02:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:00.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:00 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:03:00.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:00 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T03:03:00.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:00 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:03:00.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:00 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T03:03:00.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:00 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:03:00.401 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:00 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T03:03:00.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:03:00.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T03:03:00.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi151.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:03:00.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:00 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi151.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T03:03:00.526 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:03:00.527 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":68,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:02:50.457972+0000","last_up_change":"2024-04-05T03:01:11.145768+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T03:01:42.593775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T03:02:47.380696+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T03:02:47.698394+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6803","nonce":122630893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6805","nonce":122630893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6809","nonce":122630893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6807","nonce":122630893}]},"public_addr":"172.21.15.33:6803/122630893","cluster_addr":"172.21.15.33:6805/122630893","heartbeat_back_addr":"172.21.15.33:6809/122630893","heartbeat_front_addr":"172.21.15.33:6807/122630893","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6811","nonce":2920348148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6813","nonce":2920348148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6817","nonce":2920348148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6815","nonce":2920348148}]},"public_addr":"172.21.15.33:6811/2920348148","cluster_addr":"172.21.15.33:6813/2920348148","heartbeat_back_addr":"172.21.15.33:6817/2920348148","heartbeat_front_addr":"172.21.15.33:6815/2920348148","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6819","nonce":3206964518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6821","nonce":3206964518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6825","nonce":3206964518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6823","nonce":3206964518}]},"public_addr":"172.21.15.33:6819/3206964518","cluster_addr":"172.21.15.33:6821/3206964518","heartbeat_back_addr":"172.21.15.33:6825/3206964518","heartbeat_front_addr":"172.21.15.33:6823/3206964518","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6827","nonce":1608936270}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6829","nonce":1608936270}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6833","nonce":1608936270}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6831","nonce":1608936270}]},"public_addr":"172.21.15.33:6827/1608936270","cluster_addr":"172.21.15.33:6829/1608936270","heartbeat_back_addr":"172.21.15.33:6833/1608936270","heartbeat_front_addr":"172.21.15.33:6831/1608936270","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3239504883":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/2211132569":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/1272921316":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/1632976587":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:03:01.222 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-05T03:03:01.223 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:03:01.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:01 smithi033 ceph-mon[29244]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:01.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:01 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3952000732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:03:01.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:01 smithi033 ceph-mon[40008]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:01.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:01 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3952000732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:03:01.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:01 smithi151 ceph-mon[32789]: pgmap v47: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:01.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:01 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3952000732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:03:01.816 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:03.142 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:02 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:03.142 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:02 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:03.142 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:02 smithi033 ceph-mon[40008]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:03.142 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:02 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:03.142 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:02 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:03.143 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:02 smithi033 ceph-mon[29244]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:03.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:02 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:03.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:02 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:03.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:02 smithi151 ceph-mon[32789]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:03.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:03:03 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:03:03.225Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003649137s 2024-04-05T03:03:03.662 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:03:03.663 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":68,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:02:50.457972+0000","last_up_change":"2024-04-05T03:01:11.145768+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T03:01:42.593775+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T03:02:47.380696+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T03:02:47.698394+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6803","nonce":122630893}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6805","nonce":122630893}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6809","nonce":122630893}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":122630893},{"type":"v1","addr":"172.21.15.33:6807","nonce":122630893}]},"public_addr":"172.21.15.33:6803/122630893","cluster_addr":"172.21.15.33:6805/122630893","heartbeat_back_addr":"172.21.15.33:6809/122630893","heartbeat_front_addr":"172.21.15.33:6807/122630893","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6811","nonce":2920348148}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6813","nonce":2920348148}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6817","nonce":2920348148}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2920348148},{"type":"v1","addr":"172.21.15.33:6815","nonce":2920348148}]},"public_addr":"172.21.15.33:6811/2920348148","cluster_addr":"172.21.15.33:6813/2920348148","heartbeat_back_addr":"172.21.15.33:6817/2920348148","heartbeat_front_addr":"172.21.15.33:6815/2920348148","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6819","nonce":3206964518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6821","nonce":3206964518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6825","nonce":3206964518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3206964518},{"type":"v1","addr":"172.21.15.33:6823","nonce":3206964518}]},"public_addr":"172.21.15.33:6819/3206964518","cluster_addr":"172.21.15.33:6821/3206964518","heartbeat_back_addr":"172.21.15.33:6825/3206964518","heartbeat_front_addr":"172.21.15.33:6823/3206964518","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6827","nonce":1608936270}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6829","nonce":1608936270}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6833","nonce":1608936270}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1608936270},{"type":"v1","addr":"172.21.15.33:6831","nonce":1608936270}]},"public_addr":"172.21.15.33:6827/1608936270","cluster_addr":"172.21.15.33:6829/1608936270","heartbeat_back_addr":"172.21.15.33:6833/1608936270","heartbeat_front_addr":"172.21.15.33:6831/1608936270","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3239504883":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/2211132569":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/1272921316":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/1632976587":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:03:04.191 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:03 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1170117521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:03:04.191 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:03 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1170117521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:03:04.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:03 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1170117521' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:03:04.838 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-05T03:03:04.838 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-05T03:03:04.839 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-05T03:03:04.839 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-05T03:03:04.839 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-05T03:03:04.840 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-05T03:03:04.840 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-05T03:03:04.841 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-05T03:03:05.210 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:05 smithi033 ceph-mon[40008]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:05.211 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:05 smithi033 ceph-mon[29244]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:05.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:05 smithi151 ceph-mon[32789]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:05.853 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:05.863 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:05.900 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:05.920 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:05.935 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:05.940 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:06.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:06 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:06.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:06 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:06.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:06 smithi033 ceph-mon[29244]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:06.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:06 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:06.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:06 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:06.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:06 smithi033 ceph-mon[40008]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:07.022 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:06 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:07.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:06 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:07.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:06 smithi151 ceph-mon[32789]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:07.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:03:07 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:03:07.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:07.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T03:03:07.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:07 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:08.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:07 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:08.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:07 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:08.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:07 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:08.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:07 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:03:08.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:07 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T03:03:08.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:07 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:08.877 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:08.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:08 smithi033 ceph-mon[40008]: Deploying daemon mds.foo.smithi033.rmzwzs on smithi033 2024-04-05T03:03:08.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:08 smithi033 ceph-mon[40008]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:08 smithi033 ceph-mon[29244]: Deploying daemon mds.foo.smithi033.rmzwzs on smithi033 2024-04-05T03:03:08.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:08 smithi033 ceph-mon[29244]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:08 smithi151 ceph-mon[32789]: Deploying daemon mds.foo.smithi033.rmzwzs on smithi033 2024-04-05T03:03:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:08 smithi151 ceph-mon[32789]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:11.449 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:11 smithi033 ceph-mon[29244]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:11.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:11 smithi033 ceph-mon[40008]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:11.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:11 smithi151 ceph-mon[32789]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:12.290 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:12.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:03:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:12] "GET /metrics HTTP/1.1" 200 208205 "" "Prometheus/2.43.0" 2024-04-05T03:03:13.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:13 smithi033 ceph-mon[29244]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:13.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:13 smithi033 ceph-mon[40008]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:13 smithi151 ceph-mon[32789]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:14.172 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:14 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:14.172 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:14 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:14.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:14 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:15.149 INFO:teuthology.orchestra.run.smithi033.stdout:167503724580 2024-04-05T03:03:15.149 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-05T03:03:15.445 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:15 smithi033 ceph-mon[40008]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:15.445 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:15 smithi033 ceph-mon[29244]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:15 smithi151 ceph-mon[32789]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:17 smithi151 ceph-mon[32789]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:17.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:03:17 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:03:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:17 smithi033 ceph-mon[29244]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:17.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:17 smithi033 ceph-mon[40008]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:18.813 INFO:teuthology.orchestra.run.smithi033.stdout:201863462945 2024-04-05T03:03:18.814 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-05T03:03:19.353 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:19 smithi033 ceph-mon[40008]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:19.353 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:19 smithi033 ceph-mon[29244]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:19 smithi151 ceph-mon[32789]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:19.585 INFO:teuthology.orchestra.run.smithi033.stdout:77309411379 2024-04-05T03:03:19.585 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-05T03:03:20.436 INFO:teuthology.orchestra.run.smithi033.stdout:236223201308 2024-04-05T03:03:20.437 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-05T03:03:21.012 INFO:teuthology.orchestra.run.smithi033.stdout:111669149743 2024-04-05T03:03:21.012 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-05T03:03:21.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:21 smithi033 ceph-mon[29244]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:21.424 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:21 smithi033 ceph-mon[40008]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:21.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:21 smithi151 ceph-mon[32789]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:21.690 INFO:teuthology.orchestra.run.smithi033.stdout:133143986219 2024-04-05T03:03:21.690 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-05T03:03:22.605 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:03:22 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:22] "GET /metrics HTTP/1.1" 200 208181 "" "Prometheus/2.43.0" 2024-04-05T03:03:23.381 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:23 smithi033 ceph-mon[29244]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:23.381 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:23 smithi033 ceph-mon[40008]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:23.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:23 smithi151 ceph-mon[32789]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:24.042 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:24.042 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:24.078 INFO:teuthology.orchestra.run.smithi033.stdout:34359738431 2024-04-05T03:03:24.078 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-05T03:03:25.404 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:25 smithi033 ceph-mon[29244]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:25.404 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:25 smithi033 ceph-mon[40008]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:25.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:25 smithi151 ceph-mon[32789]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:25.783 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:25.783 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:25.783 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:25.784 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:25.820 INFO:teuthology.orchestra.run.smithi033.stdout:55834574906 2024-04-05T03:03:25.820 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-05T03:03:27.193 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:27 smithi033 ceph-mon[40008]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:27.193 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:27 smithi033 ceph-mon[29244]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:27.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:27 smithi151 ceph-mon[32789]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:27.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:03:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:03:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: daemon mds.foo.smithi033.rmzwzs assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: Cluster is now healthy 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:boot 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:creating} 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: daemon mds.foo.smithi033.rmzwzs is now active in filesystem foo as rank 0 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: daemon mds.foo.smithi033.rmzwzs assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-05T03:03:29.433 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: Cluster is now healthy 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:boot 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:creating} 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: daemon mds.foo.smithi033.rmzwzs is now active in filesystem foo as rank 0 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:29.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: daemon mds.foo.smithi033.rmzwzs assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: Cluster is now healthy 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:boot 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:creating} 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: daemon mds.foo.smithi033.rmzwzs is now active in filesystem foo as rank 0 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:29.435 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:30.198 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:30 smithi033 ceph-mon[29244]: Deploying daemon mds.foo.smithi151.cxpowy on smithi151 2024-04-05T03:03:30.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:30 smithi033 ceph-mon[29244]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:active 2024-04-05T03:03:30.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:30 smithi033 ceph-mon[29244]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 2024-04-05T03:03:30.199 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:30 smithi033 ceph-mon[40008]: Deploying daemon mds.foo.smithi151.cxpowy on smithi151 2024-04-05T03:03:30.199 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:30 smithi033 ceph-mon[40008]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:active 2024-04-05T03:03:30.199 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:30 smithi033 ceph-mon[40008]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 2024-04-05T03:03:30.404 INFO:teuthology.orchestra.run.smithi033.stdout:111669149745 2024-04-05T03:03:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:30 smithi151 ceph-mon[32789]: Deploying daemon mds.foo.smithi151.cxpowy on smithi151 2024-04-05T03:03:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:30 smithi151 ceph-mon[32789]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:active 2024-04-05T03:03:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:30 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 2024-04-05T03:03:31.206 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[40008]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:31.207 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3928274873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:03:31.207 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.207 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.207 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.207 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.207 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.207 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[29244]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:31.208 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3928274873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:03:31.208 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.208 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.208 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.208 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.208 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:31 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:31 smithi151 ceph-mon[32789]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:31 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3928274873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:03:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:31 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:31 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:31 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:31 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:31 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:32.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:32] "GET /metrics HTTP/1.1" 200 238606 "" "Prometheus/2.43.0" 2024-04-05T03:03:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[29244]: mds.? [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] up:boot 2024-04-05T03:03:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[29244]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:03:32.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[29244]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:32.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:32.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[40008]: mds.? [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] up:boot 2024-04-05T03:03:32.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[40008]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:32.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:03:32.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:32 smithi033 ceph-mon[40008]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:32.516 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:32 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:32.516 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:32 smithi151 ceph-mon[32789]: mds.? [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] up:boot 2024-04-05T03:03:32.516 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:32 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:32.516 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:32 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:03:32.516 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:32 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:32.870 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:32.900 INFO:teuthology.orchestra.run.smithi033.stdout:133143986221 2024-04-05T03:03:33.115 INFO:teuthology.orchestra.run.smithi033.stdout:201863462948 2024-04-05T03:03:33.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:33 smithi151 ceph-mon[32789]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:33.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:33 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3412895009' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:03:33.472 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:33 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3542327835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:03:33.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:33 smithi033 ceph-mon[29244]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:33.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:33 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3412895009' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:03:33.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:33 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3542327835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:03:33.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:33 smithi033 ceph-mon[40008]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:33.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:33 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3412895009' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:03:33.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:33 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3542327835' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:03:33.854 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462945 got 201863462948 for osd.6 2024-04-05T03:03:33.855 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:34.260 INFO:teuthology.orchestra.run.smithi033.stdout:236223201311 2024-04-05T03:03:34.842 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149743 got 111669149745 for osd.3 2024-04-05T03:03:34.842 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:34.914 INFO:teuthology.orchestra.run.smithi033.stdout:167503724584 2024-04-05T03:03:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[29244]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:active 2024-04-05T03:03:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[29244]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[29244]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2345115667' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[40008]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:active 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[40008]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[40008]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2345115667' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:34 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:34 smithi151 ceph-mon[32789]: mds.? [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] up:active 2024-04-05T03:03:35.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:34 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:35.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:34 smithi151 ceph-mon[32789]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:35.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:34 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:34 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2345115667' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:03:35.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:34 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:34 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:35.639 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986219 got 133143986221 for osd.4 2024-04-05T03:03:35.640 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:36.205 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:35 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3064200533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:03:36.206 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:35 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3064200533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:03:36.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:35 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3064200533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:03:36.710 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724580 got 167503724584 for osd.5 2024-04-05T03:03:36.711 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:36.974 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:36.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:36 smithi033 ceph-mon[40008]: mds.? [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] up:standby 2024-04-05T03:03:36.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:36 smithi033 ceph-mon[40008]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:36.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:36 smithi033 ceph-mon[40008]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:36.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:36 smithi033 ceph-mon[29244]: mds.? [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] up:standby 2024-04-05T03:03:36.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:36 smithi033 ceph-mon[29244]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:36.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:36 smithi033 ceph-mon[29244]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:37.117 INFO:teuthology.orchestra.run.smithi033.stdout:77309411383 2024-04-05T03:03:37.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:03:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:03:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:36 smithi151 ceph-mon[32789]: mds.? [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] up:standby 2024-04-05T03:03:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:36 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:03:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:36 smithi151 ceph-mon[32789]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:37.429 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201308 got 236223201311 for osd.7 2024-04-05T03:03:37.429 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:37.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:37 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2635009393' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:03:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:37 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2635009393' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:03:38.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:37 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2635009393' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:03:38.364 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411379 got 77309411383 for osd.2 2024-04-05T03:03:38.364 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:39.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:38 smithi033 ceph-mon[40008]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:39.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:38 smithi033 ceph-mon[29244]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:38 smithi151 ceph-mon[32789]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:40.380 INFO:teuthology.orchestra.run.smithi033.stdout:55834574910 2024-04-05T03:03:40.973 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574906 got 55834574910 for osd.1 2024-04-05T03:03:40.973 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:41.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:41 smithi033 ceph-mon[29244]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:41.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:41 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3730521591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:03:41.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:41 smithi033 ceph-mon[40008]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:41.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:41 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3730521591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:03:41.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:41 smithi151 ceph-mon[32789]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T03:03:41.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:41 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3730521591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:03:41.997 INFO:teuthology.orchestra.run.smithi033.stdout:34359738435 2024-04-05T03:03:42.292 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:42 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2228165174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:03:42.292 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:42 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2228165174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:03:42.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:42 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2228165174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:03:42.580 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:03:42 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:42] "GET /metrics HTTP/1.1" 200 238606 "" "Prometheus/2.43.0" 2024-04-05T03:03:43.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:43 smithi033 ceph-mon[40008]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-05T03:03:43.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:43 smithi033 ceph-mon[29244]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-05T03:03:43.359 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738431 got 34359738435 for osd.0 2024-04-05T03:03:43.359 DEBUG:teuthology.parallel:result is None 2024-04-05T03:03:43.359 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-05T03:03:43.360 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:03:43.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:43 smithi151 ceph-mon[32789]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-05T03:03:44.038 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:44.400 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:44.401 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T03:03:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-05T03:03:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-05T03:03:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:45.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-05T03:03:45.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:45.371 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: osdmap e69: 8 total, 8 up, 8 in 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-04-05T03:03:45.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-05T03:03:45.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: osdmap e69: 8 total, 8 up, 8 in 2024-04-05T03:03:45.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:45.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:45.373 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:45 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: osdmap e69: 8 total, 8 up, 8 in 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:45.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:45.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:45 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:03:45.828 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:03:46.395 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:46 smithi151 ceph-mon[32789]: osdmap e70: 8 total, 8 up, 8 in 2024-04-05T03:03:46.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:46 smithi033 ceph-mon[29244]: osdmap e70: 8 total, 8 up, 8 in 2024-04-05T03:03:46.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:46 smithi033 ceph-mon[40008]: osdmap e70: 8 total, 8 up, 8 in 2024-04-05T03:03:46.592 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":71,"stamp":"2024-04-05T03:03:45.130574+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386148,"kb_used_data":7300,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575068,"statfs":{"total":767960285184,"available":765516869632,"internally_reserved":0,"allocated":7475200,"data_stored":4203249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.075912"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143393+0000","last_change":"2024-04-05T03:02:48.902639+0000","last_active":"2024-04-05T03:02:52.143393+0000","last_peered":"2024-04-05T03:02:52.143393+0000","last_clean":"2024-04-05T03:02:52.143393+0000","last_became_active":"2024-04-05T03:02:48.902301+0000","last_became_peered":"2024-04-05T03:02:48.902301+0000","last_unstale":"2024-04-05T03:02:52.143393+0000","last_undegraded":"2024-04-05T03:02:52.143393+0000","last_fullsized":"2024-04-05T03:02:52.143393+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:20:53.245725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863502+0000","last_change":"2024-04-05T03:02:48.457889+0000","last_active":"2024-04-05T03:03:28.863502+0000","last_peered":"2024-04-05T03:03:28.863502+0000","last_clean":"2024-04-05T03:03:28.863502+0000","last_became_active":"2024-04-05T03:02:48.457391+0000","last_became_peered":"2024-04-05T03:02:48.457391+0000","last_unstale":"2024-04-05T03:03:28.863502+0000","last_undegraded":"2024-04-05T03:03:28.863502+0000","last_fullsized":"2024-04-05T03:03:28.863502+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:38:13.647069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471599+0000","last_change":"2024-04-05T03:02:49.746985+0000","last_active":"2024-04-05T03:02:50.471599+0000","last_peered":"2024-04-05T03:02:50.471599+0000","last_clean":"2024-04-05T03:02:50.471599+0000","last_became_active":"2024-04-05T03:02:49.746610+0000","last_became_peered":"2024-04-05T03:02:49.746610+0000","last_unstale":"2024-04-05T03:02:50.471599+0000","last_undegraded":"2024-04-05T03:02:50.471599+0000","last_fullsized":"2024-04-05T03:02:50.471599+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:56:20.326917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862607+0000","last_change":"2024-04-05T03:02:48.459650+0000","last_active":"2024-04-05T03:03:28.862607+0000","last_peered":"2024-04-05T03:03:28.862607+0000","last_clean":"2024-04-05T03:03:28.862607+0000","last_became_active":"2024-04-05T03:02:48.459355+0000","last_became_peered":"2024-04-05T03:02:48.459355+0000","last_unstale":"2024-04-05T03:03:28.862607+0000","last_undegraded":"2024-04-05T03:03:28.862607+0000","last_fullsized":"2024-04-05T03:03:28.862607+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:23:16.774935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143737+0000","last_change":"2024-04-05T03:02:48.644045+0000","last_active":"2024-04-05T03:02:52.143737+0000","last_peered":"2024-04-05T03:02:52.143737+0000","last_clean":"2024-04-05T03:02:52.143737+0000","last_became_active":"2024-04-05T03:02:48.643795+0000","last_became_peered":"2024-04-05T03:02:48.643795+0000","last_unstale":"2024-04-05T03:02:52.143737+0000","last_undegraded":"2024-04-05T03:02:52.143737+0000","last_fullsized":"2024-04-05T03:02:52.143737+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:49:15.865276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863111+0000","last_change":"2024-04-05T03:02:48.458396+0000","last_active":"2024-04-05T03:03:28.863111+0000","last_peered":"2024-04-05T03:03:28.863111+0000","last_clean":"2024-04-05T03:03:28.863111+0000","last_became_active":"2024-04-05T03:02:48.458092+0000","last_became_peered":"2024-04-05T03:02:48.458092+0000","last_unstale":"2024-04-05T03:03:28.863111+0000","last_undegraded":"2024-04-05T03:03:28.863111+0000","last_fullsized":"2024-04-05T03:03:28.863111+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:43.973088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144020+0000","last_change":"2024-04-05T03:02:48.472069+0000","last_active":"2024-04-05T03:02:52.144020+0000","last_peered":"2024-04-05T03:02:52.144020+0000","last_clean":"2024-04-05T03:02:52.144020+0000","last_became_active":"2024-04-05T03:02:48.471739+0000","last_became_peered":"2024-04-05T03:02:48.471739+0000","last_unstale":"2024-04-05T03:02:52.144020+0000","last_undegraded":"2024-04-05T03:02:52.144020+0000","last_fullsized":"2024-04-05T03:02:52.144020+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:44:49.173752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863132+0000","last_change":"2024-04-05T03:02:48.459793+0000","last_active":"2024-04-05T03:03:28.863132+0000","last_peered":"2024-04-05T03:03:28.863132+0000","last_clean":"2024-04-05T03:03:28.863132+0000","last_became_active":"2024-04-05T03:02:48.459518+0000","last_became_peered":"2024-04-05T03:02:48.459518+0000","last_unstale":"2024-04-05T03:03:28.863132+0000","last_undegraded":"2024-04-05T03:03:28.863132+0000","last_fullsized":"2024-04-05T03:03:28.863132+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:03:23.429419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.145044+0000","last_change":"2024-04-05T03:02:48.473124+0000","last_active":"2024-04-05T03:02:51.145044+0000","last_peered":"2024-04-05T03:02:51.145044+0000","last_clean":"2024-04-05T03:02:51.145044+0000","last_became_active":"2024-04-05T03:02:48.472816+0000","last_became_peered":"2024-04-05T03:02:48.472816+0000","last_unstale":"2024-04-05T03:02:51.145044+0000","last_undegraded":"2024-04-05T03:02:51.145044+0000","last_fullsized":"2024-04-05T03:02:51.145044+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:09:10.287592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.861099+0000","last_change":"2024-04-05T03:02:48.457652+0000","last_active":"2024-04-05T03:03:28.861099+0000","last_peered":"2024-04-05T03:03:28.861099+0000","last_clean":"2024-04-05T03:03:28.861099+0000","last_became_active":"2024-04-05T03:02:48.457340+0000","last_became_peered":"2024-04-05T03:02:48.457340+0000","last_unstale":"2024-04-05T03:03:28.861099+0000","last_undegraded":"2024-04-05T03:03:28.861099+0000","last_fullsized":"2024-04-05T03:03:28.861099+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:04:12.013393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144379+0000","last_change":"2024-04-05T03:02:48.902933+0000","last_active":"2024-04-05T03:02:52.144379+0000","last_peered":"2024-04-05T03:02:52.144379+0000","last_clean":"2024-04-05T03:02:52.144379+0000","last_became_active":"2024-04-05T03:02:48.902583+0000","last_became_peered":"2024-04-05T03:02:48.902583+0000","last_unstale":"2024-04-05T03:02:52.144379+0000","last_undegraded":"2024-04-05T03:02:52.144379+0000","last_fullsized":"2024-04-05T03:02:52.144379+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:11:03.431547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:33.157906+0000","last_change":"2024-04-05T03:02:48.465598+0000","last_active":"2024-04-05T03:03:33.157906+0000","last_peered":"2024-04-05T03:03:33.157906+0000","last_clean":"2024-04-05T03:03:33.157906+0000","last_became_active":"2024-04-05T03:02:48.465280+0000","last_became_peered":"2024-04-05T03:02:48.465280+0000","last_unstale":"2024-04-05T03:03:33.157906+0000","last_undegraded":"2024-04-05T03:03:33.157906+0000","last_fullsized":"2024-04-05T03:03:33.157906+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:27:18.537900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144239+0000","last_change":"2024-04-05T03:01:43.659234+0000","last_active":"2024-04-05T03:02:52.144239+0000","last_peered":"2024-04-05T03:02:52.144239+0000","last_clean":"2024-04-05T03:02:52.144239+0000","last_became_active":"2024-04-05T03:01:43.658801+0000","last_became_peered":"2024-04-05T03:01:43.658801+0000","last_unstale":"2024-04-05T03:02:52.144239+0000","last_undegraded":"2024-04-05T03:02:52.144239+0000","last_fullsized":"2024-04-05T03:02:52.144239+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:39:37.401263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470713+0000","last_change":"2024-04-05T03:02:50.003751+0000","last_active":"2024-04-05T03:02:50.470713+0000","last_peered":"2024-04-05T03:02:50.470713+0000","last_clean":"2024-04-05T03:02:50.470713+0000","last_became_active":"2024-04-05T03:02:50.003472+0000","last_became_peered":"2024-04-05T03:02:50.003472+0000","last_unstale":"2024-04-05T03:02:50.470713+0000","last_undegraded":"2024-04-05T03:02:50.470713+0000","last_fullsized":"2024-04-05T03:02:50.470713+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:46:41.938159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851325+0000","last_change":"2024-04-05T03:01:43.656498+0000","last_active":"2024-04-05T03:02:50.851325+0000","last_peered":"2024-04-05T03:02:50.851325+0000","last_clean":"2024-04-05T03:02:50.851325+0000","last_became_active":"2024-04-05T03:01:43.656159+0000","last_became_peered":"2024-04-05T03:01:43.656159+0000","last_unstale":"2024-04-05T03:02:50.851325+0000","last_undegraded":"2024-04-05T03:02:50.851325+0000","last_fullsized":"2024-04-05T03:02:50.851325+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:39.619074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862930+0000","last_change":"2024-04-05T03:02:48.458703+0000","last_active":"2024-04-05T03:03:28.862930+0000","last_peered":"2024-04-05T03:03:28.862930+0000","last_clean":"2024-04-05T03:03:28.862930+0000","last_became_active":"2024-04-05T03:02:48.457630+0000","last_became_peered":"2024-04-05T03:02:48.457630+0000","last_unstale":"2024-04-05T03:03:28.862930+0000","last_undegraded":"2024-04-05T03:03:28.862930+0000","last_fullsized":"2024-04-05T03:03:28.862930+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:42:21.323311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470743+0000","last_change":"2024-04-05T03:02:50.002775+0000","last_active":"2024-04-05T03:02:50.470743+0000","last_peered":"2024-04-05T03:02:50.470743+0000","last_clean":"2024-04-05T03:02:50.470743+0000","last_became_active":"2024-04-05T03:02:50.002356+0000","last_became_peered":"2024-04-05T03:02:50.002356+0000","last_unstale":"2024-04-05T03:02:50.470743+0000","last_undegraded":"2024-04-05T03:02:50.470743+0000","last_fullsized":"2024-04-05T03:02:50.470743+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:47:10.699733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851341+0000","last_change":"2024-04-05T03:02:48.456621+0000","last_active":"2024-04-05T03:02:50.851341+0000","last_peered":"2024-04-05T03:02:50.851341+0000","last_clean":"2024-04-05T03:02:50.851341+0000","last_became_active":"2024-04-05T03:02:48.455491+0000","last_became_peered":"2024-04-05T03:02:48.455491+0000","last_unstale":"2024-04-05T03:02:50.851341+0000","last_undegraded":"2024-04-05T03:02:50.851341+0000","last_fullsized":"2024-04-05T03:02:50.851341+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:38.283998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.329024+0000","last_change":"2024-04-05T03:01:43.910231+0000","last_active":"2024-04-05T03:02:51.329024+0000","last_peered":"2024-04-05T03:02:51.329024+0000","last_clean":"2024-04-05T03:02:51.329024+0000","last_became_active":"2024-04-05T03:01:43.909886+0000","last_became_peered":"2024-04-05T03:01:43.909886+0000","last_unstale":"2024-04-05T03:02:51.329024+0000","last_undegraded":"2024-04-05T03:02:51.329024+0000","last_fullsized":"2024-04-05T03:02:51.329024+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:22:51.464507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471123+0000","last_change":"2024-04-05T03:02:49.746307+0000","last_active":"2024-04-05T03:02:50.471123+0000","last_peered":"2024-04-05T03:02:50.471123+0000","last_clean":"2024-04-05T03:02:50.471123+0000","last_became_active":"2024-04-05T03:02:49.745976+0000","last_became_peered":"2024-04-05T03:02:49.745976+0000","last_unstale":"2024-04-05T03:02:50.471123+0000","last_undegraded":"2024-04-05T03:02:50.471123+0000","last_fullsized":"2024-04-05T03:02:50.471123+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:49:15.142317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471245+0000","last_change":"2024-04-05T03:02:48.461046+0000","last_active":"2024-04-05T03:02:50.471245+0000","last_peered":"2024-04-05T03:02:50.471245+0000","last_clean":"2024-04-05T03:02:50.471245+0000","last_became_active":"2024-04-05T03:02:48.460678+0000","last_became_peered":"2024-04-05T03:02:48.460678+0000","last_unstale":"2024-04-05T03:02:50.471245+0000","last_undegraded":"2024-04-05T03:02:50.471245+0000","last_fullsized":"2024-04-05T03:02:50.471245+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:23.864420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.328885+0000","last_change":"2024-04-05T03:01:43.910268+0000","last_active":"2024-04-05T03:02:51.328885+0000","last_peered":"2024-04-05T03:02:51.328885+0000","last_clean":"2024-04-05T03:02:51.328885+0000","last_became_active":"2024-04-05T03:01:43.909991+0000","last_became_peered":"2024-04-05T03:01:43.909991+0000","last_unstale":"2024-04-05T03:02:51.328885+0000","last_undegraded":"2024-04-05T03:02:51.328885+0000","last_fullsized":"2024-04-05T03:02:51.328885+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:12.008125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144169+0000","last_change":"2024-04-05T03:02:50.003073+0000","last_active":"2024-04-05T03:02:52.144169+0000","last_peered":"2024-04-05T03:02:52.144169+0000","last_clean":"2024-04-05T03:02:52.144169+0000","last_became_active":"2024-04-05T03:02:50.002726+0000","last_became_peered":"2024-04-05T03:02:50.002726+0000","last_unstale":"2024-04-05T03:02:52.144169+0000","last_undegraded":"2024-04-05T03:02:52.144169+0000","last_fullsized":"2024-04-05T03:02:52.144169+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:50:50.549731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471063+0000","last_change":"2024-04-05T03:02:48.460577+0000","last_active":"2024-04-05T03:02:50.471063+0000","last_peered":"2024-04-05T03:02:50.471063+0000","last_clean":"2024-04-05T03:02:50.471063+0000","last_became_active":"2024-04-05T03:02:48.460216+0000","last_became_peered":"2024-04-05T03:02:48.460216+0000","last_unstale":"2024-04-05T03:02:50.471063+0000","last_undegraded":"2024-04-05T03:02:50.471063+0000","last_fullsized":"2024-04-05T03:02:50.471063+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:35:58.345475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":276,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.328579+0000","last_change":"2024-04-05T03:01:14.837104+0000","last_active":"2024-04-05T03:02:51.328579+0000","last_peered":"2024-04-05T03:02:51.328579+0000","last_clean":"2024-04-05T03:02:51.328579+0000","last_became_active":"2024-04-05T03:01:14.836690+0000","last_became_peered":"2024-04-05T03:01:14.836690+0000","last_unstale":"2024-04-05T03:02:51.328579+0000","last_undegraded":"2024-04-05T03:02:51.328579+0000","last_fullsized":"2024-04-05T03:02:51.328579+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:48:28.544381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470393+0000","last_change":"2024-04-05T03:01:43.659091+0000","last_active":"2024-04-05T03:02:50.470393+0000","last_peered":"2024-04-05T03:02:50.470393+0000","last_clean":"2024-04-05T03:02:50.470393+0000","last_became_active":"2024-04-05T03:01:43.658783+0000","last_became_peered":"2024-04-05T03:01:43.658783+0000","last_unstale":"2024-04-05T03:02:50.470393+0000","last_undegraded":"2024-04-05T03:02:50.470393+0000","last_fullsized":"2024-04-05T03:02:50.470393+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:34:20.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851283+0000","last_change":"2024-04-05T03:02:49.773210+0000","last_active":"2024-04-05T03:02:50.851283+0000","last_peered":"2024-04-05T03:02:50.851283+0000","last_clean":"2024-04-05T03:02:50.851283+0000","last_became_active":"2024-04-05T03:02:49.772812+0000","last_became_peered":"2024-04-05T03:02:49.772812+0000","last_unstale":"2024-04-05T03:02:50.851283+0000","last_undegraded":"2024-04-05T03:02:50.851283+0000","last_fullsized":"2024-04-05T03:02:50.851283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:29:41.425482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850960+0000","last_change":"2024-04-05T03:02:48.457064+0000","last_active":"2024-04-05T03:02:50.850960+0000","last_peered":"2024-04-05T03:02:50.850960+0000","last_clean":"2024-04-05T03:02:50.850960+0000","last_became_active":"2024-04-05T03:02:48.456787+0000","last_became_peered":"2024-04-05T03:02:48.456787+0000","last_unstale":"2024-04-05T03:02:50.850960+0000","last_undegraded":"2024-04-05T03:02:50.850960+0000","last_fullsized":"2024-04-05T03:02:50.850960+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:01:04.753701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.144942+0000","last_change":"2024-04-05T03:01:43.910452+0000","last_active":"2024-04-05T03:02:51.144942+0000","last_peered":"2024-04-05T03:02:51.144942+0000","last_clean":"2024-04-05T03:02:51.144942+0000","last_became_active":"2024-04-05T03:01:43.909876+0000","last_became_peered":"2024-04-05T03:01:43.909876+0000","last_unstale":"2024-04-05T03:02:51.144942+0000","last_undegraded":"2024-04-05T03:02:51.144942+0000","last_fullsized":"2024-04-05T03:02:51.144942+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:07:36.880582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851281+0000","last_change":"2024-04-05T03:02:49.746957+0000","last_active":"2024-04-05T03:02:50.851281+0000","last_peered":"2024-04-05T03:02:50.851281+0000","last_clean":"2024-04-05T03:02:50.851281+0000","last_became_active":"2024-04-05T03:02:49.746064+0000","last_became_peered":"2024-04-05T03:02:49.746064+0000","last_unstale":"2024-04-05T03:02:50.851281+0000","last_undegraded":"2024-04-05T03:02:50.851281+0000","last_fullsized":"2024-04-05T03:02:50.851281+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:42:42.677796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143923+0000","last_change":"2024-04-05T03:02:48.455210+0000","last_active":"2024-04-05T03:02:52.143923+0000","last_peered":"2024-04-05T03:02:52.143923+0000","last_clean":"2024-04-05T03:02:52.143923+0000","last_became_active":"2024-04-05T03:02:48.453370+0000","last_became_peered":"2024-04-05T03:02:48.453370+0000","last_unstale":"2024-04-05T03:02:52.143923+0000","last_undegraded":"2024-04-05T03:02:52.143923+0000","last_fullsized":"2024-04-05T03:02:52.143923+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:50.364653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470386+0000","last_change":"2024-04-05T03:01:43.656346+0000","last_active":"2024-04-05T03:02:50.470386+0000","last_peered":"2024-04-05T03:02:50.470386+0000","last_clean":"2024-04-05T03:02:50.470386+0000","last_became_active":"2024-04-05T03:01:43.656045+0000","last_became_peered":"2024-04-05T03:01:43.656045+0000","last_unstale":"2024-04-05T03:02:50.470386+0000","last_undegraded":"2024-04-05T03:02:50.470386+0000","last_fullsized":"2024-04-05T03:02:50.470386+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:40:55.612040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850947+0000","last_change":"2024-04-05T03:02:49.772316+0000","last_active":"2024-04-05T03:02:50.850947+0000","last_peered":"2024-04-05T03:02:50.850947+0000","last_clean":"2024-04-05T03:02:50.850947+0000","last_became_active":"2024-04-05T03:02:49.772060+0000","last_became_peered":"2024-04-05T03:02:49.772060+0000","last_unstale":"2024-04-05T03:02:50.850947+0000","last_undegraded":"2024-04-05T03:02:50.850947+0000","last_fullsized":"2024-04-05T03:02:50.850947+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:05:14.265331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470573+0000","last_change":"2024-04-05T03:02:48.457200+0000","last_active":"2024-04-05T03:02:50.470573+0000","last_peered":"2024-04-05T03:02:50.470573+0000","last_clean":"2024-04-05T03:02:50.470573+0000","last_became_active":"2024-04-05T03:02:48.456346+0000","last_became_peered":"2024-04-05T03:02:48.456346+0000","last_unstale":"2024-04-05T03:02:50.470573+0000","last_undegraded":"2024-04-05T03:02:50.470573+0000","last_fullsized":"2024-04-05T03:02:50.470573+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:22.908070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851247+0000","last_change":"2024-04-05T03:01:43.910340+0000","last_active":"2024-04-05T03:02:50.851247+0000","last_peered":"2024-04-05T03:02:50.851247+0000","last_clean":"2024-04-05T03:02:50.851247+0000","last_became_active":"2024-04-05T03:01:43.909987+0000","last_became_peered":"2024-04-05T03:01:43.909987+0000","last_unstale":"2024-04-05T03:02:50.851247+0000","last_undegraded":"2024-04-05T03:02:50.851247+0000","last_fullsized":"2024-04-05T03:02:50.851247+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:42.918552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851260+0000","last_change":"2024-04-05T03:02:49.469399+0000","last_active":"2024-04-05T03:02:50.851260+0000","last_peered":"2024-04-05T03:02:50.851260+0000","last_clean":"2024-04-05T03:02:50.851260+0000","last_became_active":"2024-04-05T03:02:49.469112+0000","last_became_peered":"2024-04-05T03:02:49.469112+0000","last_unstale":"2024-04-05T03:02:50.851260+0000","last_undegraded":"2024-04-05T03:02:50.851260+0000","last_fullsized":"2024-04-05T03:02:50.851260+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:59:24.415770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470803+0000","last_change":"2024-04-05T03:02:48.460772+0000","last_active":"2024-04-05T03:02:50.470803+0000","last_peered":"2024-04-05T03:02:50.470803+0000","last_clean":"2024-04-05T03:02:50.470803+0000","last_became_active":"2024-04-05T03:02:48.458509+0000","last_became_peered":"2024-04-05T03:02:48.458509+0000","last_unstale":"2024-04-05T03:02:50.470803+0000","last_undegraded":"2024-04-05T03:02:50.470803+0000","last_fullsized":"2024-04-05T03:02:50.470803+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:56:41.129273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143849+0000","last_change":"2024-04-05T03:02:48.902626+0000","last_active":"2024-04-05T03:02:52.143849+0000","last_peered":"2024-04-05T03:02:52.143849+0000","last_clean":"2024-04-05T03:02:52.143849+0000","last_became_active":"2024-04-05T03:02:48.902271+0000","last_became_peered":"2024-04-05T03:02:48.902271+0000","last_unstale":"2024-04-05T03:02:52.143849+0000","last_undegraded":"2024-04-05T03:02:52.143849+0000","last_fullsized":"2024-04-05T03:02:52.143849+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:31:05.812000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143666+0000","last_change":"2024-04-05T03:02:48.448680+0000","last_active":"2024-04-05T03:02:52.143666+0000","last_peered":"2024-04-05T03:02:52.143666+0000","last_clean":"2024-04-05T03:02:52.143666+0000","last_became_active":"2024-04-05T03:02:48.448318+0000","last_became_peered":"2024-04-05T03:02:48.448318+0000","last_unstale":"2024-04-05T03:02:52.143666+0000","last_undegraded":"2024-04-05T03:02:52.143666+0000","last_fullsized":"2024-04-05T03:02:52.143666+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:17.475717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144390+0000","last_change":"2024-04-05T03:02:48.472192+0000","last_active":"2024-04-05T03:02:52.144390+0000","last_peered":"2024-04-05T03:02:52.144390+0000","last_clean":"2024-04-05T03:02:52.144390+0000","last_became_active":"2024-04-05T03:02:48.471904+0000","last_became_peered":"2024-04-05T03:02:48.471904+0000","last_unstale":"2024-04-05T03:02:52.144390+0000","last_undegraded":"2024-04-05T03:02:52.144390+0000","last_fullsized":"2024-04-05T03:02:52.144390+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:38:38.241850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143993+0000","last_change":"2024-04-05T03:02:48.459300+0000","last_active":"2024-04-05T03:02:52.143993+0000","last_peered":"2024-04-05T03:02:52.143993+0000","last_clean":"2024-04-05T03:02:52.143993+0000","last_became_active":"2024-04-05T03:02:48.458886+0000","last_became_peered":"2024-04-05T03:02:48.458886+0000","last_unstale":"2024-04-05T03:02:52.143993+0000","last_undegraded":"2024-04-05T03:02:52.143993+0000","last_fullsized":"2024-04-05T03:02:52.143993+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:55:07.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143990+0000","last_change":"2024-04-05T03:02:48.904058+0000","last_active":"2024-04-05T03:02:52.143990+0000","last_peered":"2024-04-05T03:02:52.143990+0000","last_clean":"2024-04-05T03:02:52.143990+0000","last_became_active":"2024-04-05T03:02:48.903756+0000","last_became_peered":"2024-04-05T03:02:48.903756+0000","last_unstale":"2024-04-05T03:02:52.143990+0000","last_undegraded":"2024-04-05T03:02:52.143990+0000","last_fullsized":"2024-04-05T03:02:52.143990+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:11:54.123968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.861865+0000","last_change":"2024-04-05T03:02:48.457251+0000","last_active":"2024-04-05T03:03:28.861865+0000","last_peered":"2024-04-05T03:03:28.861865+0000","last_clean":"2024-04-05T03:03:28.861865+0000","last_became_active":"2024-04-05T03:02:48.456950+0000","last_became_peered":"2024-04-05T03:02:48.456950+0000","last_unstale":"2024-04-05T03:03:28.861865+0000","last_undegraded":"2024-04-05T03:03:28.861865+0000","last_fullsized":"2024-04-05T03:03:28.861865+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:56:54.797317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470634+0000","last_change":"2024-04-05T03:02:49.473606+0000","last_active":"2024-04-05T03:02:50.470634+0000","last_peered":"2024-04-05T03:02:50.470634+0000","last_clean":"2024-04-05T03:02:50.470634+0000","last_became_active":"2024-04-05T03:02:49.473288+0000","last_became_peered":"2024-04-05T03:02:49.473288+0000","last_unstale":"2024-04-05T03:02:50.470634+0000","last_undegraded":"2024-04-05T03:02:50.470634+0000","last_fullsized":"2024-04-05T03:02:50.470634+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:16:58.265293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470398+0000","last_change":"2024-04-05T03:02:48.461876+0000","last_active":"2024-04-05T03:02:50.470398+0000","last_peered":"2024-04-05T03:02:50.470398+0000","last_clean":"2024-04-05T03:02:50.470398+0000","last_became_active":"2024-04-05T03:02:48.461576+0000","last_became_peered":"2024-04-05T03:02:48.461576+0000","last_unstale":"2024-04-05T03:02:50.470398+0000","last_undegraded":"2024-04-05T03:02:50.470398+0000","last_fullsized":"2024-04-05T03:02:50.470398+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:24:53.545330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-05T03:03:44.124565+0000","last_change":"2024-04-05T03:03:44.124707+0000","last_active":"2024-04-05T03:03:44.124544+0000","last_peered":"2024-04-05T03:02:52.143812+0000","last_clean":"2024-04-05T03:02:52.143812+0000","last_became_active":"2024-04-05T03:02:50.002029+0000","last_became_peered":"2024-04-05T03:02:50.002029+0000","last_unstale":"2024-04-05T03:03:44.124565+0000","last_undegraded":"2024-04-05T03:03:44.124565+0000","last_fullsized":"2024-04-05T03:03:44.124565+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:46:24.791221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862713+0000","last_change":"2024-04-05T03:02:48.461252+0000","last_active":"2024-04-05T03:03:28.862713+0000","last_peered":"2024-04-05T03:03:28.862713+0000","last_clean":"2024-04-05T03:03:28.862713+0000","last_became_active":"2024-04-05T03:02:48.460982+0000","last_became_peered":"2024-04-05T03:02:48.460982+0000","last_unstale":"2024-04-05T03:03:28.862713+0000","last_undegraded":"2024-04-05T03:03:28.862713+0000","last_fullsized":"2024-04-05T03:03:28.862713+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:03.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143502+0000","last_change":"2024-04-05T03:02:48.472006+0000","last_active":"2024-04-05T03:02:52.143502+0000","last_peered":"2024-04-05T03:02:52.143502+0000","last_clean":"2024-04-05T03:02:52.143502+0000","last_became_active":"2024-04-05T03:02:48.471696+0000","last_became_peered":"2024-04-05T03:02:48.471696+0000","last_unstale":"2024-04-05T03:02:52.143502+0000","last_undegraded":"2024-04-05T03:02:52.143502+0000","last_fullsized":"2024-04-05T03:02:52.143502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:31:37.135819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863374+0000","last_change":"2024-04-05T03:02:48.457001+0000","last_active":"2024-04-05T03:03:28.863374+0000","last_peered":"2024-04-05T03:03:28.863374+0000","last_clean":"2024-04-05T03:03:28.863374+0000","last_became_active":"2024-04-05T03:02:48.456334+0000","last_became_peered":"2024-04-05T03:02:48.456334+0000","last_unstale":"2024-04-05T03:03:28.863374+0000","last_undegraded":"2024-04-05T03:03:28.863374+0000","last_fullsized":"2024-04-05T03:03:28.863374+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:16:57.078617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470262+0000","last_change":"2024-04-05T03:02:48.903134+0000","last_active":"2024-04-05T03:02:50.470262+0000","last_peered":"2024-04-05T03:02:50.470262+0000","last_clean":"2024-04-05T03:02:50.470262+0000","last_became_active":"2024-04-05T03:02:48.902836+0000","last_became_peered":"2024-04-05T03:02:48.902836+0000","last_unstale":"2024-04-05T03:02:50.470262+0000","last_undegraded":"2024-04-05T03:02:50.470262+0000","last_fullsized":"2024-04-05T03:02:50.470262+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:08:54.250101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863324+0000","last_change":"2024-04-05T03:02:48.457358+0000","last_active":"2024-04-05T03:03:28.863324+0000","last_peered":"2024-04-05T03:03:28.863324+0000","last_clean":"2024-04-05T03:03:28.863324+0000","last_became_active":"2024-04-05T03:02:48.456987+0000","last_became_peered":"2024-04-05T03:02:48.456987+0000","last_unstale":"2024-04-05T03:03:28.863324+0000","last_undegraded":"2024-04-05T03:03:28.863324+0000","last_fullsized":"2024-04-05T03:03:28.863324+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:22.057798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471639+0000","last_change":"2024-04-05T03:02:50.003001+0000","last_active":"2024-04-05T03:02:50.471639+0000","last_peered":"2024-04-05T03:02:50.471639+0000","last_clean":"2024-04-05T03:02:50.471639+0000","last_became_active":"2024-04-05T03:02:50.002734+0000","last_became_peered":"2024-04-05T03:02:50.002734+0000","last_unstale":"2024-04-05T03:02:50.471639+0000","last_undegraded":"2024-04-05T03:02:50.471639+0000","last_fullsized":"2024-04-05T03:02:50.471639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:52:55.958187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.864004+0000","last_change":"2024-04-05T03:02:48.457497+0000","last_active":"2024-04-05T03:03:28.864004+0000","last_peered":"2024-04-05T03:03:28.864004+0000","last_clean":"2024-04-05T03:03:28.864004+0000","last_became_active":"2024-04-05T03:02:48.457161+0000","last_became_peered":"2024-04-05T03:02:48.457161+0000","last_unstale":"2024-04-05T03:03:28.864004+0000","last_undegraded":"2024-04-05T03:03:28.864004+0000","last_fullsized":"2024-04-05T03:03:28.864004+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:31:49.521313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470859+0000","last_change":"2024-04-05T03:02:50.003556+0000","last_active":"2024-04-05T03:02:50.470859+0000","last_peered":"2024-04-05T03:02:50.470859+0000","last_clean":"2024-04-05T03:02:50.470859+0000","last_became_active":"2024-04-05T03:02:50.003271+0000","last_became_peered":"2024-04-05T03:02:50.003271+0000","last_unstale":"2024-04-05T03:02:50.470859+0000","last_undegraded":"2024-04-05T03:02:50.470859+0000","last_fullsized":"2024-04-05T03:02:50.470859+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:00:59.126639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.328713+0000","last_change":"2024-04-05T03:02:48.460853+0000","last_active":"2024-04-05T03:02:51.328713+0000","last_peered":"2024-04-05T03:02:51.328713+0000","last_clean":"2024-04-05T03:02:51.328713+0000","last_became_active":"2024-04-05T03:02:48.460557+0000","last_became_peered":"2024-04-05T03:02:48.460557+0000","last_unstale":"2024-04-05T03:02:51.328713+0000","last_undegraded":"2024-04-05T03:02:51.328713+0000","last_fullsized":"2024-04-05T03:02:51.328713+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:57.261993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470571+0000","last_change":"2024-04-05T03:02:48.902752+0000","last_active":"2024-04-05T03:02:50.470571+0000","last_peered":"2024-04-05T03:02:50.470571+0000","last_clean":"2024-04-05T03:02:50.470571+0000","last_became_active":"2024-04-05T03:02:48.902396+0000","last_became_peered":"2024-04-05T03:02:48.902396+0000","last_unstale":"2024-04-05T03:02:50.470571+0000","last_undegraded":"2024-04-05T03:02:50.470571+0000","last_fullsized":"2024-04-05T03:02:50.470571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:45:00.227236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470519+0000","last_change":"2024-04-05T03:02:48.461320+0000","last_active":"2024-04-05T03:02:50.470519+0000","last_peered":"2024-04-05T03:02:50.470519+0000","last_clean":"2024-04-05T03:02:50.470519+0000","last_became_active":"2024-04-05T03:02:48.461034+0000","last_became_peered":"2024-04-05T03:02:48.461034+0000","last_unstale":"2024-04-05T03:02:50.470519+0000","last_undegraded":"2024-04-05T03:02:50.470519+0000","last_fullsized":"2024-04-05T03:02:50.470519+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:34:08.199664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471422+0000","last_change":"2024-04-05T03:02:50.002729+0000","last_active":"2024-04-05T03:02:50.471422+0000","last_peered":"2024-04-05T03:02:50.471422+0000","last_clean":"2024-04-05T03:02:50.471422+0000","last_became_active":"2024-04-05T03:02:50.002342+0000","last_became_peered":"2024-04-05T03:02:50.002342+0000","last_unstale":"2024-04-05T03:02:50.471422+0000","last_undegraded":"2024-04-05T03:02:50.471422+0000","last_fullsized":"2024-04-05T03:02:50.471422+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:51:56.223511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863597+0000","last_change":"2024-04-05T03:02:48.459896+0000","last_active":"2024-04-05T03:03:28.863597+0000","last_peered":"2024-04-05T03:03:28.863597+0000","last_clean":"2024-04-05T03:03:28.863597+0000","last_became_active":"2024-04-05T03:02:48.458108+0000","last_became_peered":"2024-04-05T03:02:48.458108+0000","last_unstale":"2024-04-05T03:03:28.863597+0000","last_undegraded":"2024-04-05T03:03:28.863597+0000","last_fullsized":"2024-04-05T03:03:28.863597+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:41:12.902400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144181+0000","last_change":"2024-04-05T03:02:48.903143+0000","last_active":"2024-04-05T03:02:52.144181+0000","last_peered":"2024-04-05T03:02:52.144181+0000","last_clean":"2024-04-05T03:02:52.144181+0000","last_became_active":"2024-04-05T03:02:48.902265+0000","last_became_peered":"2024-04-05T03:02:48.902265+0000","last_unstale":"2024-04-05T03:02:52.144181+0000","last_undegraded":"2024-04-05T03:02:52.144181+0000","last_fullsized":"2024-04-05T03:02:52.144181+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:05:56.179576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.865403+0000","last_change":"2024-04-05T03:02:48.455833+0000","last_active":"2024-04-05T03:03:28.865403+0000","last_peered":"2024-04-05T03:03:28.865403+0000","last_clean":"2024-04-05T03:03:28.865403+0000","last_became_active":"2024-04-05T03:02:48.455185+0000","last_became_peered":"2024-04-05T03:02:48.455185+0000","last_unstale":"2024-04-05T03:03:28.865403+0000","last_undegraded":"2024-04-05T03:03:28.865403+0000","last_fullsized":"2024-04-05T03:03:28.865403+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:02.621579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850940+0000","last_change":"2024-04-05T03:02:49.746606+0000","last_active":"2024-04-05T03:02:50.850940+0000","last_peered":"2024-04-05T03:02:50.850940+0000","last_clean":"2024-04-05T03:02:50.850940+0000","last_became_active":"2024-04-05T03:02:49.746308+0000","last_became_peered":"2024-04-05T03:02:49.746308+0000","last_unstale":"2024-04-05T03:02:50.850940+0000","last_undegraded":"2024-04-05T03:02:50.850940+0000","last_fullsized":"2024-04-05T03:02:50.850940+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:07:39.965218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.864133+0000","last_change":"2024-04-05T03:02:48.456697+0000","last_active":"2024-04-05T03:03:28.864133+0000","last_peered":"2024-04-05T03:03:28.864133+0000","last_clean":"2024-04-05T03:03:28.864133+0000","last_became_active":"2024-04-05T03:02:48.456140+0000","last_became_peered":"2024-04-05T03:02:48.456140+0000","last_unstale":"2024-04-05T03:03:28.864133+0000","last_undegraded":"2024-04-05T03:03:28.864133+0000","last_fullsized":"2024-04-05T03:03:28.864133+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:57:31.290707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471616+0000","last_change":"2024-04-05T03:02:50.002658+0000","last_active":"2024-04-05T03:02:50.471616+0000","last_peered":"2024-04-05T03:02:50.471616+0000","last_clean":"2024-04-05T03:02:50.471616+0000","last_became_active":"2024-04-05T03:02:50.002383+0000","last_became_peered":"2024-04-05T03:02:50.002383+0000","last_unstale":"2024-04-05T03:02:50.471616+0000","last_undegraded":"2024-04-05T03:02:50.471616+0000","last_fullsized":"2024-04-05T03:02:50.471616+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:05:34.117134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470514+0000","last_change":"2024-04-05T03:02:48.457005+0000","last_active":"2024-04-05T03:02:50.470514+0000","last_peered":"2024-04-05T03:02:50.470514+0000","last_clean":"2024-04-05T03:02:50.470514+0000","last_became_active":"2024-04-05T03:02:48.456670+0000","last_became_peered":"2024-04-05T03:02:48.456670+0000","last_unstale":"2024-04-05T03:02:50.470514+0000","last_undegraded":"2024-04-05T03:02:50.470514+0000","last_fullsized":"2024-04-05T03:02:50.470514+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:45:12.114782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470894+0000","last_change":"2024-04-05T03:02:49.471218+0000","last_active":"2024-04-05T03:02:50.470894+0000","last_peered":"2024-04-05T03:02:50.470894+0000","last_clean":"2024-04-05T03:02:50.470894+0000","last_became_active":"2024-04-05T03:02:49.470918+0000","last_became_peered":"2024-04-05T03:02:49.470918+0000","last_unstale":"2024-04-05T03:02:50.470894+0000","last_undegraded":"2024-04-05T03:02:50.470894+0000","last_fullsized":"2024-04-05T03:02:50.470894+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:48:19.807934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471027+0000","last_change":"2024-04-05T03:02:48.463560+0000","last_active":"2024-04-05T03:02:50.471027+0000","last_peered":"2024-04-05T03:02:50.471027+0000","last_clean":"2024-04-05T03:02:50.471027+0000","last_became_active":"2024-04-05T03:02:48.463245+0000","last_became_peered":"2024-04-05T03:02:48.463245+0000","last_unstale":"2024-04-05T03:02:50.471027+0000","last_undegraded":"2024-04-05T03:02:50.471027+0000","last_fullsized":"2024-04-05T03:02:50.471027+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:47:16.914625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-05T03:03:44.124416+0000","last_change":"2024-04-05T03:03:44.124631+0000","last_active":"2024-04-05T03:03:44.124380+0000","last_peered":"2024-04-05T03:02:52.144105+0000","last_clean":"2024-04-05T03:02:52.144105+0000","last_became_active":"2024-04-05T03:02:49.772945+0000","last_became_peered":"2024-04-05T03:02:49.772945+0000","last_unstale":"2024-04-05T03:03:44.124416+0000","last_undegraded":"2024-04-05T03:03:44.124416+0000","last_fullsized":"2024-04-05T03:03:44.124416+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:20:53.755431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470983+0000","last_change":"2024-04-05T03:02:48.457628+0000","last_active":"2024-04-05T03:02:50.470983+0000","last_peered":"2024-04-05T03:02:50.470983+0000","last_clean":"2024-04-05T03:02:50.470983+0000","last_became_active":"2024-04-05T03:02:48.457318+0000","last_became_peered":"2024-04-05T03:02:48.457318+0000","last_unstale":"2024-04-05T03:02:50.470983+0000","last_undegraded":"2024-04-05T03:02:50.470983+0000","last_fullsized":"2024-04-05T03:02:50.470983+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:32.231625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470942+0000","last_change":"2024-04-05T03:02:50.004105+0000","last_active":"2024-04-05T03:02:50.470942+0000","last_peered":"2024-04-05T03:02:50.470942+0000","last_clean":"2024-04-05T03:02:50.470942+0000","last_became_active":"2024-04-05T03:02:50.003693+0000","last_became_peered":"2024-04-05T03:02:50.003693+0000","last_unstale":"2024-04-05T03:02:50.470942+0000","last_undegraded":"2024-04-05T03:02:50.470942+0000","last_fullsized":"2024-04-05T03:02:50.470942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:02.044564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850887+0000","last_change":"2024-04-05T03:02:48.464554+0000","last_active":"2024-04-05T03:02:50.850887+0000","last_peered":"2024-04-05T03:02:50.850887+0000","last_clean":"2024-04-05T03:02:50.850887+0000","last_became_active":"2024-04-05T03:02:48.463923+0000","last_became_peered":"2024-04-05T03:02:48.463923+0000","last_unstale":"2024-04-05T03:02:50.850887+0000","last_undegraded":"2024-04-05T03:02:50.850887+0000","last_fullsized":"2024-04-05T03:02:50.850887+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:56.699626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-05T03:03:44.123283+0000","last_change":"2024-04-05T03:03:44.123504+0000","last_active":"2024-04-05T03:03:44.123251+0000","last_peered":"2024-04-05T03:02:50.471482+0000","last_clean":"2024-04-05T03:02:50.471482+0000","last_became_active":"2024-04-05T03:02:49.471004+0000","last_became_peered":"2024-04-05T03:02:49.471004+0000","last_unstale":"2024-04-05T03:03:44.123283+0000","last_undegraded":"2024-04-05T03:03:44.123283+0000","last_fullsized":"2024-04-05T03:03:44.123283+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:39:58.591265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4,5],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862264+0000","last_change":"2024-04-05T03:02:48.456415+0000","last_active":"2024-04-05T03:03:28.862264+0000","last_peered":"2024-04-05T03:03:28.862264+0000","last_clean":"2024-04-05T03:03:28.862264+0000","last_became_active":"2024-04-05T03:02:48.455959+0000","last_became_peered":"2024-04-05T03:02:48.455959+0000","last_unstale":"2024-04-05T03:03:28.862264+0000","last_undegraded":"2024-04-05T03:03:28.862264+0000","last_fullsized":"2024-04-05T03:03:28.862264+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:52:33.266219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201314,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1323008,"data_stored":895341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.38400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51000000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.504}]},{"osd":1,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78700000000000003,"5min":0.878,"15min":0.878},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.379,"5min":0.371,"15min":0.371},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.48699999999999999}]},{"osd":2,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.41699999999999998}]},{"osd":3,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55400000000000005,"15min":0.55300000000000005},"min":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72999999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.44500000000000001}]},{"osd":4,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48199999999999998,"15min":0.48399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70099999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53300000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.502,"15min":0.50600000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.77400000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77100000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56799999999999995}]},{"osd":6,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.39800000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.432,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78800000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.55800000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462951,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":894951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35199999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.623},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.50600000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72599999999999998,"5min":0.88,"15min":0.88},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.51500000000000001,"15min":0.51900000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.58699999999999997}]},{"osd":3,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.40600000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76000000000000001,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.36499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77700000000000002,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.47899999999999998}]},{"osd":4,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.42999999999999999,"15min":0.42699999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.432,"15min":0.41999999999999998},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.42899999999999999}]},{"osd":5,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.504,"15min":0.49199999999999999},"min":{"1min":0.36299999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.52600000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56399999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574911,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":302940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.312},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.374,"5min":0.379,"15min":0.34499999999999997},"min":{"1min":0.26100000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.54900000000000004,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.32600000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:03:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40600000000000003,"15min":0.39400000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.621,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.38},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.39200000000000002,"15min":0.37},"min":{"1min":0.24199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53800000000000003,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.42999999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.42099999999999999,"15min":0.42199999999999999},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69599999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.32400000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.9610000000000001,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.40699999999999997}]},{"osd":4,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63200000000000001,"15min":0.63900000000000001},"min":{"1min":0.45600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.85399999999999998,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86199999999999999,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.53800000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:29 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64100000000000001,"15min":0.64300000000000002},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93999999999999995,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":0.59599999999999997}]},{"osd":6,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.628,"15min":0.621},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.629,"15min":0.63600000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76500000000000001,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.505}]},{"osd":7,"last update":"Fri Apr 5 03:03:13 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59999999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.659,"5min":1.659,"15min":1.659},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.48999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738436,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298628,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446524,"statfs":{"total":95995035648,"available":95689240576,"internally_reserved":0,"allocated":1302528,"data_stored":894165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 03:02:56 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35399999999999998,"15min":0.32400000000000001},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":0.53700000000000003,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.374,"15min":0.34000000000000002},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.54200000000000004,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.33600000000000002}]},{"osd":2,"last update":"Fri Apr 5 03:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.46300000000000002,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.54000000000000004,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.376}]},{"osd":3,"last update":"Fri Apr 5 03:02:44 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.35499999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.59899999999999998,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.45300000000000001,"15min":0.45400000000000001},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59599999999999997,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.434}]},{"osd":4,"last update":"Fri Apr 5 03:03:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.875,"15min":0.875},"last":0.66000000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.41699999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79500000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83599999999999997,"5min":1.018,"15min":1.018},"last":0.61499999999999999}]},{"osd":6,"last update":"Fri Apr 5 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.42499999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81499999999999995,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.748},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64600000000000002,"15min":0.64500000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.64000000000000001}]},{"osd":7,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63500000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74099999999999999,"5min":0.875,"15min":0.875},"last":0.63},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.69699999999999995,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80600000000000005,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.83299999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":303809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51000000000000001,"5min":0.623,"15min":0.623},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39000000000000001,"15min":0.39400000000000002},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.53000000000000003,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.29299999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.39500000000000002,"15min":0.40500000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57599999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.54200000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39900000000000002}]},{"osd":3,"last update":"Fri Apr 5 03:02:44 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.45900000000000002,"15min":0.46400000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.70299999999999996,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.45000000000000001,"15min":0.45800000000000002},"min":{"1min":0.313,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.33900000000000002}]},{"osd":4,"last update":"Fri Apr 5 03:03:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.41299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.83699999999999997,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":12.170999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86799999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.56699999999999995}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80800000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.66600000000000004,"15min":0.60899999999999999},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":12.968999999999999,"5min":12.968999999999999,"15min":12.968999999999999},"last":0.57799999999999996}]},{"osd":6,"last update":"Fri Apr 5 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.83399999999999996,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.45400000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73599999999999999,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.61699999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:03:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.63900000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149749,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298044,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447108,"statfs":{"total":95995035648,"available":95689838592,"internally_reserved":0,"allocated":704512,"data_stored":303295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.39000000000000001,"15min":0.38900000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.439,"15min":0.44500000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.5740000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.34899999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.45900000000000002,"15min":0.45100000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.45600000000000002,"15min":0.45900000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":0.32600000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.30399999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.5009999999999999,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.441}]},{"osd":4,"last update":"Fri Apr 5 03:03:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.434,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.96899999999999997,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.63100000000000001}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.432,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":1.002,"15min":1.002},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63600000000000001,"15min":0.64300000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.67500000000000004}]},{"osd":6,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.63900000000000001,"15min":0.64700000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.873,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89400000000000002,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.61699999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.63700000000000001,"15min":0.64100000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46800000000000003,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.65600000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986224,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":304903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.56100000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54700000000000004,"15min":0.56000000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66700000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.47499999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.753,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.499},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55200000000000005,"15min":0.56200000000000006},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.67700000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.59399999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.64600000000000002,"15min":0.57299999999999995},"min":{"1min":0.372,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":11.239000000000001,"5min":11.239000000000001,"15min":11.239000000000001},"last":0.63},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57199999999999995,"15min":0.58999999999999997},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69499999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52000000000000002}]},{"osd":3,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.67900000000000005,"15min":0.61499999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.133,"5min":0.69099999999999995,"15min":0.625},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.221,"5min":11.221,"15min":11.221},"last":0.61899999999999999}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.40200000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.432},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.623,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.45100000000000001}]},{"osd":6,"last update":"Fri Apr 5 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.45500000000000002,"15min":0.44700000000000001},"min":{"1min":0.34399999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.376},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45700000000000002,"15min":0.44900000000000001},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66900000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.54800000000000004}]},{"osd":7,"last update":"Fri Apr 5 03:03:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.49299999999999999,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.58599999999999997,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.438,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.35899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.56999999999999995,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.436}]}]},{"osd":5,"up_from":39,"seq":167503724587,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":303845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74399999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.44600000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.39600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.78400000000000003,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.63700000000000001,"15min":0.58099999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.2070000000000007,"5min":9.2070000000000007,"15min":9.2070000000000007},"last":0.495}]},{"osd":2,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.443,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.55100000000000005,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.81599999999999995,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.70999999999999996}]},{"osd":3,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.75,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.56699999999999995,"15min":0.55700000000000005},"min":{"1min":0.437,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78100000000000003,"5min":1.329,"15min":1.329},"last":0.68300000000000005}]},{"osd":4,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.40899999999999997,"15min":0.38100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.42999999999999999,"15min":0.40200000000000002},"min":{"1min":0.32300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.71499999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.58199999999999996}]},{"osd":6,"last update":"Fri Apr 5 03:02:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.46800000000000003,"15min":0.45900000000000002},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.47999999999999998,"15min":0.46600000000000003},"min":{"1min":0.38,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.66400000000000003}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81999999999999995,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77000000000000002,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.622}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:03:46.596 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:03:46.924 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:47.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:47 smithi151 ceph-mon[32789]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:47.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:47 smithi151 ceph-mon[32789]: pgmap v72: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:47.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:03:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:03:47.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:47 smithi033 ceph-mon[29244]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:47.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:47 smithi033 ceph-mon[29244]: pgmap v72: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:47.302 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:47 smithi033 ceph-mon[40008]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:47.302 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:47 smithi033 ceph-mon[40008]: pgmap v72: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:48.745 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:03:48.746 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:03:48.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:48 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:48.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:48 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:48.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:48 smithi033 ceph-mon[29244]: pgmap v73: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:48.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:48 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:48.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:48 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:48.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:48 smithi033 ceph-mon[40008]: pgmap v73: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:49.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:48 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:49.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:48 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:49.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:48 smithi151 ceph-mon[32789]: pgmap v73: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:49.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:49 smithi033 ceph-mon[29244]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:49.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:49 smithi033 ceph-mon[40008]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:50.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:49 smithi151 ceph-mon[32789]: from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:50.185 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":73,"stamp":"2024-04-05T03:03:48.057429+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386148,"kb_used_data":7300,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575068,"statfs":{"total":767960285184,"available":765516869632,"internally_reserved":0,"allocated":7475200,"data_stored":4203249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001660"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143393+0000","last_change":"2024-04-05T03:02:48.902639+0000","last_active":"2024-04-05T03:02:52.143393+0000","last_peered":"2024-04-05T03:02:52.143393+0000","last_clean":"2024-04-05T03:02:52.143393+0000","last_became_active":"2024-04-05T03:02:48.902301+0000","last_became_peered":"2024-04-05T03:02:48.902301+0000","last_unstale":"2024-04-05T03:02:52.143393+0000","last_undegraded":"2024-04-05T03:02:52.143393+0000","last_fullsized":"2024-04-05T03:02:52.143393+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:20:53.245725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863502+0000","last_change":"2024-04-05T03:02:48.457889+0000","last_active":"2024-04-05T03:03:28.863502+0000","last_peered":"2024-04-05T03:03:28.863502+0000","last_clean":"2024-04-05T03:03:28.863502+0000","last_became_active":"2024-04-05T03:02:48.457391+0000","last_became_peered":"2024-04-05T03:02:48.457391+0000","last_unstale":"2024-04-05T03:03:28.863502+0000","last_undegraded":"2024-04-05T03:03:28.863502+0000","last_fullsized":"2024-04-05T03:03:28.863502+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:38:13.647069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471599+0000","last_change":"2024-04-05T03:02:49.746985+0000","last_active":"2024-04-05T03:02:50.471599+0000","last_peered":"2024-04-05T03:02:50.471599+0000","last_clean":"2024-04-05T03:02:50.471599+0000","last_became_active":"2024-04-05T03:02:49.746610+0000","last_became_peered":"2024-04-05T03:02:49.746610+0000","last_unstale":"2024-04-05T03:02:50.471599+0000","last_undegraded":"2024-04-05T03:02:50.471599+0000","last_fullsized":"2024-04-05T03:02:50.471599+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:56:20.326917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862607+0000","last_change":"2024-04-05T03:02:48.459650+0000","last_active":"2024-04-05T03:03:28.862607+0000","last_peered":"2024-04-05T03:03:28.862607+0000","last_clean":"2024-04-05T03:03:28.862607+0000","last_became_active":"2024-04-05T03:02:48.459355+0000","last_became_peered":"2024-04-05T03:02:48.459355+0000","last_unstale":"2024-04-05T03:03:28.862607+0000","last_undegraded":"2024-04-05T03:03:28.862607+0000","last_fullsized":"2024-04-05T03:03:28.862607+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:23:16.774935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143737+0000","last_change":"2024-04-05T03:02:48.644045+0000","last_active":"2024-04-05T03:02:52.143737+0000","last_peered":"2024-04-05T03:02:52.143737+0000","last_clean":"2024-04-05T03:02:52.143737+0000","last_became_active":"2024-04-05T03:02:48.643795+0000","last_became_peered":"2024-04-05T03:02:48.643795+0000","last_unstale":"2024-04-05T03:02:52.143737+0000","last_undegraded":"2024-04-05T03:02:52.143737+0000","last_fullsized":"2024-04-05T03:02:52.143737+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:49:15.865276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863111+0000","last_change":"2024-04-05T03:02:48.458396+0000","last_active":"2024-04-05T03:03:28.863111+0000","last_peered":"2024-04-05T03:03:28.863111+0000","last_clean":"2024-04-05T03:03:28.863111+0000","last_became_active":"2024-04-05T03:02:48.458092+0000","last_became_peered":"2024-04-05T03:02:48.458092+0000","last_unstale":"2024-04-05T03:03:28.863111+0000","last_undegraded":"2024-04-05T03:03:28.863111+0000","last_fullsized":"2024-04-05T03:03:28.863111+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:43.973088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144020+0000","last_change":"2024-04-05T03:02:48.472069+0000","last_active":"2024-04-05T03:02:52.144020+0000","last_peered":"2024-04-05T03:02:52.144020+0000","last_clean":"2024-04-05T03:02:52.144020+0000","last_became_active":"2024-04-05T03:02:48.471739+0000","last_became_peered":"2024-04-05T03:02:48.471739+0000","last_unstale":"2024-04-05T03:02:52.144020+0000","last_undegraded":"2024-04-05T03:02:52.144020+0000","last_fullsized":"2024-04-05T03:02:52.144020+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:44:49.173752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863132+0000","last_change":"2024-04-05T03:02:48.459793+0000","last_active":"2024-04-05T03:03:28.863132+0000","last_peered":"2024-04-05T03:03:28.863132+0000","last_clean":"2024-04-05T03:03:28.863132+0000","last_became_active":"2024-04-05T03:02:48.459518+0000","last_became_peered":"2024-04-05T03:02:48.459518+0000","last_unstale":"2024-04-05T03:03:28.863132+0000","last_undegraded":"2024-04-05T03:03:28.863132+0000","last_fullsized":"2024-04-05T03:03:28.863132+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:03:23.429419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.145044+0000","last_change":"2024-04-05T03:02:48.473124+0000","last_active":"2024-04-05T03:02:51.145044+0000","last_peered":"2024-04-05T03:02:51.145044+0000","last_clean":"2024-04-05T03:02:51.145044+0000","last_became_active":"2024-04-05T03:02:48.472816+0000","last_became_peered":"2024-04-05T03:02:48.472816+0000","last_unstale":"2024-04-05T03:02:51.145044+0000","last_undegraded":"2024-04-05T03:02:51.145044+0000","last_fullsized":"2024-04-05T03:02:51.145044+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:09:10.287592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.861099+0000","last_change":"2024-04-05T03:02:48.457652+0000","last_active":"2024-04-05T03:03:28.861099+0000","last_peered":"2024-04-05T03:03:28.861099+0000","last_clean":"2024-04-05T03:03:28.861099+0000","last_became_active":"2024-04-05T03:02:48.457340+0000","last_became_peered":"2024-04-05T03:02:48.457340+0000","last_unstale":"2024-04-05T03:03:28.861099+0000","last_undegraded":"2024-04-05T03:03:28.861099+0000","last_fullsized":"2024-04-05T03:03:28.861099+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:04:12.013393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144379+0000","last_change":"2024-04-05T03:02:48.902933+0000","last_active":"2024-04-05T03:02:52.144379+0000","last_peered":"2024-04-05T03:02:52.144379+0000","last_clean":"2024-04-05T03:02:52.144379+0000","last_became_active":"2024-04-05T03:02:48.902583+0000","last_became_peered":"2024-04-05T03:02:48.902583+0000","last_unstale":"2024-04-05T03:02:52.144379+0000","last_undegraded":"2024-04-05T03:02:52.144379+0000","last_fullsized":"2024-04-05T03:02:52.144379+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:11:03.431547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:33.157906+0000","last_change":"2024-04-05T03:02:48.465598+0000","last_active":"2024-04-05T03:03:33.157906+0000","last_peered":"2024-04-05T03:03:33.157906+0000","last_clean":"2024-04-05T03:03:33.157906+0000","last_became_active":"2024-04-05T03:02:48.465280+0000","last_became_peered":"2024-04-05T03:02:48.465280+0000","last_unstale":"2024-04-05T03:03:33.157906+0000","last_undegraded":"2024-04-05T03:03:33.157906+0000","last_fullsized":"2024-04-05T03:03:33.157906+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:27:18.537900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144239+0000","last_change":"2024-04-05T03:01:43.659234+0000","last_active":"2024-04-05T03:02:52.144239+0000","last_peered":"2024-04-05T03:02:52.144239+0000","last_clean":"2024-04-05T03:02:52.144239+0000","last_became_active":"2024-04-05T03:01:43.658801+0000","last_became_peered":"2024-04-05T03:01:43.658801+0000","last_unstale":"2024-04-05T03:02:52.144239+0000","last_undegraded":"2024-04-05T03:02:52.144239+0000","last_fullsized":"2024-04-05T03:02:52.144239+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:39:37.401263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470713+0000","last_change":"2024-04-05T03:02:50.003751+0000","last_active":"2024-04-05T03:02:50.470713+0000","last_peered":"2024-04-05T03:02:50.470713+0000","last_clean":"2024-04-05T03:02:50.470713+0000","last_became_active":"2024-04-05T03:02:50.003472+0000","last_became_peered":"2024-04-05T03:02:50.003472+0000","last_unstale":"2024-04-05T03:02:50.470713+0000","last_undegraded":"2024-04-05T03:02:50.470713+0000","last_fullsized":"2024-04-05T03:02:50.470713+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:46:41.938159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851325+0000","last_change":"2024-04-05T03:01:43.656498+0000","last_active":"2024-04-05T03:02:50.851325+0000","last_peered":"2024-04-05T03:02:50.851325+0000","last_clean":"2024-04-05T03:02:50.851325+0000","last_became_active":"2024-04-05T03:01:43.656159+0000","last_became_peered":"2024-04-05T03:01:43.656159+0000","last_unstale":"2024-04-05T03:02:50.851325+0000","last_undegraded":"2024-04-05T03:02:50.851325+0000","last_fullsized":"2024-04-05T03:02:50.851325+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:39.619074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862930+0000","last_change":"2024-04-05T03:02:48.458703+0000","last_active":"2024-04-05T03:03:28.862930+0000","last_peered":"2024-04-05T03:03:28.862930+0000","last_clean":"2024-04-05T03:03:28.862930+0000","last_became_active":"2024-04-05T03:02:48.457630+0000","last_became_peered":"2024-04-05T03:02:48.457630+0000","last_unstale":"2024-04-05T03:03:28.862930+0000","last_undegraded":"2024-04-05T03:03:28.862930+0000","last_fullsized":"2024-04-05T03:03:28.862930+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:42:21.323311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470743+0000","last_change":"2024-04-05T03:02:50.002775+0000","last_active":"2024-04-05T03:02:50.470743+0000","last_peered":"2024-04-05T03:02:50.470743+0000","last_clean":"2024-04-05T03:02:50.470743+0000","last_became_active":"2024-04-05T03:02:50.002356+0000","last_became_peered":"2024-04-05T03:02:50.002356+0000","last_unstale":"2024-04-05T03:02:50.470743+0000","last_undegraded":"2024-04-05T03:02:50.470743+0000","last_fullsized":"2024-04-05T03:02:50.470743+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:47:10.699733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851341+0000","last_change":"2024-04-05T03:02:48.456621+0000","last_active":"2024-04-05T03:02:50.851341+0000","last_peered":"2024-04-05T03:02:50.851341+0000","last_clean":"2024-04-05T03:02:50.851341+0000","last_became_active":"2024-04-05T03:02:48.455491+0000","last_became_peered":"2024-04-05T03:02:48.455491+0000","last_unstale":"2024-04-05T03:02:50.851341+0000","last_undegraded":"2024-04-05T03:02:50.851341+0000","last_fullsized":"2024-04-05T03:02:50.851341+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:38.283998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.329024+0000","last_change":"2024-04-05T03:01:43.910231+0000","last_active":"2024-04-05T03:02:51.329024+0000","last_peered":"2024-04-05T03:02:51.329024+0000","last_clean":"2024-04-05T03:02:51.329024+0000","last_became_active":"2024-04-05T03:01:43.909886+0000","last_became_peered":"2024-04-05T03:01:43.909886+0000","last_unstale":"2024-04-05T03:02:51.329024+0000","last_undegraded":"2024-04-05T03:02:51.329024+0000","last_fullsized":"2024-04-05T03:02:51.329024+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:22:51.464507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471123+0000","last_change":"2024-04-05T03:02:49.746307+0000","last_active":"2024-04-05T03:02:50.471123+0000","last_peered":"2024-04-05T03:02:50.471123+0000","last_clean":"2024-04-05T03:02:50.471123+0000","last_became_active":"2024-04-05T03:02:49.745976+0000","last_became_peered":"2024-04-05T03:02:49.745976+0000","last_unstale":"2024-04-05T03:02:50.471123+0000","last_undegraded":"2024-04-05T03:02:50.471123+0000","last_fullsized":"2024-04-05T03:02:50.471123+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:49:15.142317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471245+0000","last_change":"2024-04-05T03:02:48.461046+0000","last_active":"2024-04-05T03:02:50.471245+0000","last_peered":"2024-04-05T03:02:50.471245+0000","last_clean":"2024-04-05T03:02:50.471245+0000","last_became_active":"2024-04-05T03:02:48.460678+0000","last_became_peered":"2024-04-05T03:02:48.460678+0000","last_unstale":"2024-04-05T03:02:50.471245+0000","last_undegraded":"2024-04-05T03:02:50.471245+0000","last_fullsized":"2024-04-05T03:02:50.471245+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:23.864420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.328885+0000","last_change":"2024-04-05T03:01:43.910268+0000","last_active":"2024-04-05T03:02:51.328885+0000","last_peered":"2024-04-05T03:02:51.328885+0000","last_clean":"2024-04-05T03:02:51.328885+0000","last_became_active":"2024-04-05T03:01:43.909991+0000","last_became_peered":"2024-04-05T03:01:43.909991+0000","last_unstale":"2024-04-05T03:02:51.328885+0000","last_undegraded":"2024-04-05T03:02:51.328885+0000","last_fullsized":"2024-04-05T03:02:51.328885+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:12.008125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144169+0000","last_change":"2024-04-05T03:02:50.003073+0000","last_active":"2024-04-05T03:02:52.144169+0000","last_peered":"2024-04-05T03:02:52.144169+0000","last_clean":"2024-04-05T03:02:52.144169+0000","last_became_active":"2024-04-05T03:02:50.002726+0000","last_became_peered":"2024-04-05T03:02:50.002726+0000","last_unstale":"2024-04-05T03:02:52.144169+0000","last_undegraded":"2024-04-05T03:02:52.144169+0000","last_fullsized":"2024-04-05T03:02:52.144169+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:50:50.549731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471063+0000","last_change":"2024-04-05T03:02:48.460577+0000","last_active":"2024-04-05T03:02:50.471063+0000","last_peered":"2024-04-05T03:02:50.471063+0000","last_clean":"2024-04-05T03:02:50.471063+0000","last_became_active":"2024-04-05T03:02:48.460216+0000","last_became_peered":"2024-04-05T03:02:48.460216+0000","last_unstale":"2024-04-05T03:02:50.471063+0000","last_undegraded":"2024-04-05T03:02:50.471063+0000","last_fullsized":"2024-04-05T03:02:50.471063+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:35:58.345475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":276,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.328579+0000","last_change":"2024-04-05T03:01:14.837104+0000","last_active":"2024-04-05T03:02:51.328579+0000","last_peered":"2024-04-05T03:02:51.328579+0000","last_clean":"2024-04-05T03:02:51.328579+0000","last_became_active":"2024-04-05T03:01:14.836690+0000","last_became_peered":"2024-04-05T03:01:14.836690+0000","last_unstale":"2024-04-05T03:02:51.328579+0000","last_undegraded":"2024-04-05T03:02:51.328579+0000","last_fullsized":"2024-04-05T03:02:51.328579+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:48:28.544381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470393+0000","last_change":"2024-04-05T03:01:43.659091+0000","last_active":"2024-04-05T03:02:50.470393+0000","last_peered":"2024-04-05T03:02:50.470393+0000","last_clean":"2024-04-05T03:02:50.470393+0000","last_became_active":"2024-04-05T03:01:43.658783+0000","last_became_peered":"2024-04-05T03:01:43.658783+0000","last_unstale":"2024-04-05T03:02:50.470393+0000","last_undegraded":"2024-04-05T03:02:50.470393+0000","last_fullsized":"2024-04-05T03:02:50.470393+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:34:20.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851283+0000","last_change":"2024-04-05T03:02:49.773210+0000","last_active":"2024-04-05T03:02:50.851283+0000","last_peered":"2024-04-05T03:02:50.851283+0000","last_clean":"2024-04-05T03:02:50.851283+0000","last_became_active":"2024-04-05T03:02:49.772812+0000","last_became_peered":"2024-04-05T03:02:49.772812+0000","last_unstale":"2024-04-05T03:02:50.851283+0000","last_undegraded":"2024-04-05T03:02:50.851283+0000","last_fullsized":"2024-04-05T03:02:50.851283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:29:41.425482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850960+0000","last_change":"2024-04-05T03:02:48.457064+0000","last_active":"2024-04-05T03:02:50.850960+0000","last_peered":"2024-04-05T03:02:50.850960+0000","last_clean":"2024-04-05T03:02:50.850960+0000","last_became_active":"2024-04-05T03:02:48.456787+0000","last_became_peered":"2024-04-05T03:02:48.456787+0000","last_unstale":"2024-04-05T03:02:50.850960+0000","last_undegraded":"2024-04-05T03:02:50.850960+0000","last_fullsized":"2024-04-05T03:02:50.850960+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:01:04.753701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.144942+0000","last_change":"2024-04-05T03:01:43.910452+0000","last_active":"2024-04-05T03:02:51.144942+0000","last_peered":"2024-04-05T03:02:51.144942+0000","last_clean":"2024-04-05T03:02:51.144942+0000","last_became_active":"2024-04-05T03:01:43.909876+0000","last_became_peered":"2024-04-05T03:01:43.909876+0000","last_unstale":"2024-04-05T03:02:51.144942+0000","last_undegraded":"2024-04-05T03:02:51.144942+0000","last_fullsized":"2024-04-05T03:02:51.144942+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:07:36.880582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851281+0000","last_change":"2024-04-05T03:02:49.746957+0000","last_active":"2024-04-05T03:02:50.851281+0000","last_peered":"2024-04-05T03:02:50.851281+0000","last_clean":"2024-04-05T03:02:50.851281+0000","last_became_active":"2024-04-05T03:02:49.746064+0000","last_became_peered":"2024-04-05T03:02:49.746064+0000","last_unstale":"2024-04-05T03:02:50.851281+0000","last_undegraded":"2024-04-05T03:02:50.851281+0000","last_fullsized":"2024-04-05T03:02:50.851281+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:42:42.677796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143923+0000","last_change":"2024-04-05T03:02:48.455210+0000","last_active":"2024-04-05T03:02:52.143923+0000","last_peered":"2024-04-05T03:02:52.143923+0000","last_clean":"2024-04-05T03:02:52.143923+0000","last_became_active":"2024-04-05T03:02:48.453370+0000","last_became_peered":"2024-04-05T03:02:48.453370+0000","last_unstale":"2024-04-05T03:02:52.143923+0000","last_undegraded":"2024-04-05T03:02:52.143923+0000","last_fullsized":"2024-04-05T03:02:52.143923+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:50.364653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470386+0000","last_change":"2024-04-05T03:01:43.656346+0000","last_active":"2024-04-05T03:02:50.470386+0000","last_peered":"2024-04-05T03:02:50.470386+0000","last_clean":"2024-04-05T03:02:50.470386+0000","last_became_active":"2024-04-05T03:01:43.656045+0000","last_became_peered":"2024-04-05T03:01:43.656045+0000","last_unstale":"2024-04-05T03:02:50.470386+0000","last_undegraded":"2024-04-05T03:02:50.470386+0000","last_fullsized":"2024-04-05T03:02:50.470386+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:40:55.612040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850947+0000","last_change":"2024-04-05T03:02:49.772316+0000","last_active":"2024-04-05T03:02:50.850947+0000","last_peered":"2024-04-05T03:02:50.850947+0000","last_clean":"2024-04-05T03:02:50.850947+0000","last_became_active":"2024-04-05T03:02:49.772060+0000","last_became_peered":"2024-04-05T03:02:49.772060+0000","last_unstale":"2024-04-05T03:02:50.850947+0000","last_undegraded":"2024-04-05T03:02:50.850947+0000","last_fullsized":"2024-04-05T03:02:50.850947+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:05:14.265331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470573+0000","last_change":"2024-04-05T03:02:48.457200+0000","last_active":"2024-04-05T03:02:50.470573+0000","last_peered":"2024-04-05T03:02:50.470573+0000","last_clean":"2024-04-05T03:02:50.470573+0000","last_became_active":"2024-04-05T03:02:48.456346+0000","last_became_peered":"2024-04-05T03:02:48.456346+0000","last_unstale":"2024-04-05T03:02:50.470573+0000","last_undegraded":"2024-04-05T03:02:50.470573+0000","last_fullsized":"2024-04-05T03:02:50.470573+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:22.908070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851247+0000","last_change":"2024-04-05T03:01:43.910340+0000","last_active":"2024-04-05T03:02:50.851247+0000","last_peered":"2024-04-05T03:02:50.851247+0000","last_clean":"2024-04-05T03:02:50.851247+0000","last_became_active":"2024-04-05T03:01:43.909987+0000","last_became_peered":"2024-04-05T03:01:43.909987+0000","last_unstale":"2024-04-05T03:02:50.851247+0000","last_undegraded":"2024-04-05T03:02:50.851247+0000","last_fullsized":"2024-04-05T03:02:50.851247+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:42.918552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.851260+0000","last_change":"2024-04-05T03:02:49.469399+0000","last_active":"2024-04-05T03:02:50.851260+0000","last_peered":"2024-04-05T03:02:50.851260+0000","last_clean":"2024-04-05T03:02:50.851260+0000","last_became_active":"2024-04-05T03:02:49.469112+0000","last_became_peered":"2024-04-05T03:02:49.469112+0000","last_unstale":"2024-04-05T03:02:50.851260+0000","last_undegraded":"2024-04-05T03:02:50.851260+0000","last_fullsized":"2024-04-05T03:02:50.851260+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:59:24.415770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470803+0000","last_change":"2024-04-05T03:02:48.460772+0000","last_active":"2024-04-05T03:02:50.470803+0000","last_peered":"2024-04-05T03:02:50.470803+0000","last_clean":"2024-04-05T03:02:50.470803+0000","last_became_active":"2024-04-05T03:02:48.458509+0000","last_became_peered":"2024-04-05T03:02:48.458509+0000","last_unstale":"2024-04-05T03:02:50.470803+0000","last_undegraded":"2024-04-05T03:02:50.470803+0000","last_fullsized":"2024-04-05T03:02:50.470803+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:56:41.129273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143849+0000","last_change":"2024-04-05T03:02:48.902626+0000","last_active":"2024-04-05T03:02:52.143849+0000","last_peered":"2024-04-05T03:02:52.143849+0000","last_clean":"2024-04-05T03:02:52.143849+0000","last_became_active":"2024-04-05T03:02:48.902271+0000","last_became_peered":"2024-04-05T03:02:48.902271+0000","last_unstale":"2024-04-05T03:02:52.143849+0000","last_undegraded":"2024-04-05T03:02:52.143849+0000","last_fullsized":"2024-04-05T03:02:52.143849+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:31:05.812000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143666+0000","last_change":"2024-04-05T03:02:48.448680+0000","last_active":"2024-04-05T03:02:52.143666+0000","last_peered":"2024-04-05T03:02:52.143666+0000","last_clean":"2024-04-05T03:02:52.143666+0000","last_became_active":"2024-04-05T03:02:48.448318+0000","last_became_peered":"2024-04-05T03:02:48.448318+0000","last_unstale":"2024-04-05T03:02:52.143666+0000","last_undegraded":"2024-04-05T03:02:52.143666+0000","last_fullsized":"2024-04-05T03:02:52.143666+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:17.475717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144390+0000","last_change":"2024-04-05T03:02:48.472192+0000","last_active":"2024-04-05T03:02:52.144390+0000","last_peered":"2024-04-05T03:02:52.144390+0000","last_clean":"2024-04-05T03:02:52.144390+0000","last_became_active":"2024-04-05T03:02:48.471904+0000","last_became_peered":"2024-04-05T03:02:48.471904+0000","last_unstale":"2024-04-05T03:02:52.144390+0000","last_undegraded":"2024-04-05T03:02:52.144390+0000","last_fullsized":"2024-04-05T03:02:52.144390+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:38:38.241850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143993+0000","last_change":"2024-04-05T03:02:48.459300+0000","last_active":"2024-04-05T03:02:52.143993+0000","last_peered":"2024-04-05T03:02:52.143993+0000","last_clean":"2024-04-05T03:02:52.143993+0000","last_became_active":"2024-04-05T03:02:48.458886+0000","last_became_peered":"2024-04-05T03:02:48.458886+0000","last_unstale":"2024-04-05T03:02:52.143993+0000","last_undegraded":"2024-04-05T03:02:52.143993+0000","last_fullsized":"2024-04-05T03:02:52.143993+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:55:07.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143990+0000","last_change":"2024-04-05T03:02:48.904058+0000","last_active":"2024-04-05T03:02:52.143990+0000","last_peered":"2024-04-05T03:02:52.143990+0000","last_clean":"2024-04-05T03:02:52.143990+0000","last_became_active":"2024-04-05T03:02:48.903756+0000","last_became_peered":"2024-04-05T03:02:48.903756+0000","last_unstale":"2024-04-05T03:02:52.143990+0000","last_undegraded":"2024-04-05T03:02:52.143990+0000","last_fullsized":"2024-04-05T03:02:52.143990+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:11:54.123968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.861865+0000","last_change":"2024-04-05T03:02:48.457251+0000","last_active":"2024-04-05T03:03:28.861865+0000","last_peered":"2024-04-05T03:03:28.861865+0000","last_clean":"2024-04-05T03:03:28.861865+0000","last_became_active":"2024-04-05T03:02:48.456950+0000","last_became_peered":"2024-04-05T03:02:48.456950+0000","last_unstale":"2024-04-05T03:03:28.861865+0000","last_undegraded":"2024-04-05T03:03:28.861865+0000","last_fullsized":"2024-04-05T03:03:28.861865+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:56:54.797317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470634+0000","last_change":"2024-04-05T03:02:49.473606+0000","last_active":"2024-04-05T03:02:50.470634+0000","last_peered":"2024-04-05T03:02:50.470634+0000","last_clean":"2024-04-05T03:02:50.470634+0000","last_became_active":"2024-04-05T03:02:49.473288+0000","last_became_peered":"2024-04-05T03:02:49.473288+0000","last_unstale":"2024-04-05T03:02:50.470634+0000","last_undegraded":"2024-04-05T03:02:50.470634+0000","last_fullsized":"2024-04-05T03:02:50.470634+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:16:58.265293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470398+0000","last_change":"2024-04-05T03:02:48.461876+0000","last_active":"2024-04-05T03:02:50.470398+0000","last_peered":"2024-04-05T03:02:50.470398+0000","last_clean":"2024-04-05T03:02:50.470398+0000","last_became_active":"2024-04-05T03:02:48.461576+0000","last_became_peered":"2024-04-05T03:02:48.461576+0000","last_unstale":"2024-04-05T03:02:50.470398+0000","last_undegraded":"2024-04-05T03:02:50.470398+0000","last_fullsized":"2024-04-05T03:02:50.470398+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:24:53.545330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-05T03:03:44.124565+0000","last_change":"2024-04-05T03:03:44.124707+0000","last_active":"2024-04-05T03:03:44.124544+0000","last_peered":"2024-04-05T03:02:52.143812+0000","last_clean":"2024-04-05T03:02:52.143812+0000","last_became_active":"2024-04-05T03:02:50.002029+0000","last_became_peered":"2024-04-05T03:02:50.002029+0000","last_unstale":"2024-04-05T03:03:44.124565+0000","last_undegraded":"2024-04-05T03:03:44.124565+0000","last_fullsized":"2024-04-05T03:03:44.124565+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:46:24.791221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862713+0000","last_change":"2024-04-05T03:02:48.461252+0000","last_active":"2024-04-05T03:03:28.862713+0000","last_peered":"2024-04-05T03:03:28.862713+0000","last_clean":"2024-04-05T03:03:28.862713+0000","last_became_active":"2024-04-05T03:02:48.460982+0000","last_became_peered":"2024-04-05T03:02:48.460982+0000","last_unstale":"2024-04-05T03:03:28.862713+0000","last_undegraded":"2024-04-05T03:03:28.862713+0000","last_fullsized":"2024-04-05T03:03:28.862713+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:03.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.143502+0000","last_change":"2024-04-05T03:02:48.472006+0000","last_active":"2024-04-05T03:02:52.143502+0000","last_peered":"2024-04-05T03:02:52.143502+0000","last_clean":"2024-04-05T03:02:52.143502+0000","last_became_active":"2024-04-05T03:02:48.471696+0000","last_became_peered":"2024-04-05T03:02:48.471696+0000","last_unstale":"2024-04-05T03:02:52.143502+0000","last_undegraded":"2024-04-05T03:02:52.143502+0000","last_fullsized":"2024-04-05T03:02:52.143502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:31:37.135819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863374+0000","last_change":"2024-04-05T03:02:48.457001+0000","last_active":"2024-04-05T03:03:28.863374+0000","last_peered":"2024-04-05T03:03:28.863374+0000","last_clean":"2024-04-05T03:03:28.863374+0000","last_became_active":"2024-04-05T03:02:48.456334+0000","last_became_peered":"2024-04-05T03:02:48.456334+0000","last_unstale":"2024-04-05T03:03:28.863374+0000","last_undegraded":"2024-04-05T03:03:28.863374+0000","last_fullsized":"2024-04-05T03:03:28.863374+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:16:57.078617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470262+0000","last_change":"2024-04-05T03:02:48.903134+0000","last_active":"2024-04-05T03:02:50.470262+0000","last_peered":"2024-04-05T03:02:50.470262+0000","last_clean":"2024-04-05T03:02:50.470262+0000","last_became_active":"2024-04-05T03:02:48.902836+0000","last_became_peered":"2024-04-05T03:02:48.902836+0000","last_unstale":"2024-04-05T03:02:50.470262+0000","last_undegraded":"2024-04-05T03:02:50.470262+0000","last_fullsized":"2024-04-05T03:02:50.470262+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:08:54.250101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863324+0000","last_change":"2024-04-05T03:02:48.457358+0000","last_active":"2024-04-05T03:03:28.863324+0000","last_peered":"2024-04-05T03:03:28.863324+0000","last_clean":"2024-04-05T03:03:28.863324+0000","last_became_active":"2024-04-05T03:02:48.456987+0000","last_became_peered":"2024-04-05T03:02:48.456987+0000","last_unstale":"2024-04-05T03:03:28.863324+0000","last_undegraded":"2024-04-05T03:03:28.863324+0000","last_fullsized":"2024-04-05T03:03:28.863324+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:22.057798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471639+0000","last_change":"2024-04-05T03:02:50.003001+0000","last_active":"2024-04-05T03:02:50.471639+0000","last_peered":"2024-04-05T03:02:50.471639+0000","last_clean":"2024-04-05T03:02:50.471639+0000","last_became_active":"2024-04-05T03:02:50.002734+0000","last_became_peered":"2024-04-05T03:02:50.002734+0000","last_unstale":"2024-04-05T03:02:50.471639+0000","last_undegraded":"2024-04-05T03:02:50.471639+0000","last_fullsized":"2024-04-05T03:02:50.471639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:52:55.958187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.864004+0000","last_change":"2024-04-05T03:02:48.457497+0000","last_active":"2024-04-05T03:03:28.864004+0000","last_peered":"2024-04-05T03:03:28.864004+0000","last_clean":"2024-04-05T03:03:28.864004+0000","last_became_active":"2024-04-05T03:02:48.457161+0000","last_became_peered":"2024-04-05T03:02:48.457161+0000","last_unstale":"2024-04-05T03:03:28.864004+0000","last_undegraded":"2024-04-05T03:03:28.864004+0000","last_fullsized":"2024-04-05T03:03:28.864004+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:31:49.521313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470859+0000","last_change":"2024-04-05T03:02:50.003556+0000","last_active":"2024-04-05T03:02:50.470859+0000","last_peered":"2024-04-05T03:02:50.470859+0000","last_clean":"2024-04-05T03:02:50.470859+0000","last_became_active":"2024-04-05T03:02:50.003271+0000","last_became_peered":"2024-04-05T03:02:50.003271+0000","last_unstale":"2024-04-05T03:02:50.470859+0000","last_undegraded":"2024-04-05T03:02:50.470859+0000","last_fullsized":"2024-04-05T03:02:50.470859+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:00:59.126639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:51.328713+0000","last_change":"2024-04-05T03:02:48.460853+0000","last_active":"2024-04-05T03:02:51.328713+0000","last_peered":"2024-04-05T03:02:51.328713+0000","last_clean":"2024-04-05T03:02:51.328713+0000","last_became_active":"2024-04-05T03:02:48.460557+0000","last_became_peered":"2024-04-05T03:02:48.460557+0000","last_unstale":"2024-04-05T03:02:51.328713+0000","last_undegraded":"2024-04-05T03:02:51.328713+0000","last_fullsized":"2024-04-05T03:02:51.328713+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:57.261993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470571+0000","last_change":"2024-04-05T03:02:48.902752+0000","last_active":"2024-04-05T03:02:50.470571+0000","last_peered":"2024-04-05T03:02:50.470571+0000","last_clean":"2024-04-05T03:02:50.470571+0000","last_became_active":"2024-04-05T03:02:48.902396+0000","last_became_peered":"2024-04-05T03:02:48.902396+0000","last_unstale":"2024-04-05T03:02:50.470571+0000","last_undegraded":"2024-04-05T03:02:50.470571+0000","last_fullsized":"2024-04-05T03:02:50.470571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:45:00.227236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470519+0000","last_change":"2024-04-05T03:02:48.461320+0000","last_active":"2024-04-05T03:02:50.470519+0000","last_peered":"2024-04-05T03:02:50.470519+0000","last_clean":"2024-04-05T03:02:50.470519+0000","last_became_active":"2024-04-05T03:02:48.461034+0000","last_became_peered":"2024-04-05T03:02:48.461034+0000","last_unstale":"2024-04-05T03:02:50.470519+0000","last_undegraded":"2024-04-05T03:02:50.470519+0000","last_fullsized":"2024-04-05T03:02:50.470519+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:34:08.199664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471422+0000","last_change":"2024-04-05T03:02:50.002729+0000","last_active":"2024-04-05T03:02:50.471422+0000","last_peered":"2024-04-05T03:02:50.471422+0000","last_clean":"2024-04-05T03:02:50.471422+0000","last_became_active":"2024-04-05T03:02:50.002342+0000","last_became_peered":"2024-04-05T03:02:50.002342+0000","last_unstale":"2024-04-05T03:02:50.471422+0000","last_undegraded":"2024-04-05T03:02:50.471422+0000","last_fullsized":"2024-04-05T03:02:50.471422+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:51:56.223511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.863597+0000","last_change":"2024-04-05T03:02:48.459896+0000","last_active":"2024-04-05T03:03:28.863597+0000","last_peered":"2024-04-05T03:03:28.863597+0000","last_clean":"2024-04-05T03:03:28.863597+0000","last_became_active":"2024-04-05T03:02:48.458108+0000","last_became_peered":"2024-04-05T03:02:48.458108+0000","last_unstale":"2024-04-05T03:03:28.863597+0000","last_undegraded":"2024-04-05T03:03:28.863597+0000","last_fullsized":"2024-04-05T03:03:28.863597+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:41:12.902400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:52.144181+0000","last_change":"2024-04-05T03:02:48.903143+0000","last_active":"2024-04-05T03:02:52.144181+0000","last_peered":"2024-04-05T03:02:52.144181+0000","last_clean":"2024-04-05T03:02:52.144181+0000","last_became_active":"2024-04-05T03:02:48.902265+0000","last_became_peered":"2024-04-05T03:02:48.902265+0000","last_unstale":"2024-04-05T03:02:52.144181+0000","last_undegraded":"2024-04-05T03:02:52.144181+0000","last_fullsized":"2024-04-05T03:02:52.144181+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:05:56.179576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.865403+0000","last_change":"2024-04-05T03:02:48.455833+0000","last_active":"2024-04-05T03:03:28.865403+0000","last_peered":"2024-04-05T03:03:28.865403+0000","last_clean":"2024-04-05T03:03:28.865403+0000","last_became_active":"2024-04-05T03:02:48.455185+0000","last_became_peered":"2024-04-05T03:02:48.455185+0000","last_unstale":"2024-04-05T03:03:28.865403+0000","last_undegraded":"2024-04-05T03:03:28.865403+0000","last_fullsized":"2024-04-05T03:03:28.865403+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:02.621579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850940+0000","last_change":"2024-04-05T03:02:49.746606+0000","last_active":"2024-04-05T03:02:50.850940+0000","last_peered":"2024-04-05T03:02:50.850940+0000","last_clean":"2024-04-05T03:02:50.850940+0000","last_became_active":"2024-04-05T03:02:49.746308+0000","last_became_peered":"2024-04-05T03:02:49.746308+0000","last_unstale":"2024-04-05T03:02:50.850940+0000","last_undegraded":"2024-04-05T03:02:50.850940+0000","last_fullsized":"2024-04-05T03:02:50.850940+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:07:39.965218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.864133+0000","last_change":"2024-04-05T03:02:48.456697+0000","last_active":"2024-04-05T03:03:28.864133+0000","last_peered":"2024-04-05T03:03:28.864133+0000","last_clean":"2024-04-05T03:03:28.864133+0000","last_became_active":"2024-04-05T03:02:48.456140+0000","last_became_peered":"2024-04-05T03:02:48.456140+0000","last_unstale":"2024-04-05T03:03:28.864133+0000","last_undegraded":"2024-04-05T03:03:28.864133+0000","last_fullsized":"2024-04-05T03:03:28.864133+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:57:31.290707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471616+0000","last_change":"2024-04-05T03:02:50.002658+0000","last_active":"2024-04-05T03:02:50.471616+0000","last_peered":"2024-04-05T03:02:50.471616+0000","last_clean":"2024-04-05T03:02:50.471616+0000","last_became_active":"2024-04-05T03:02:50.002383+0000","last_became_peered":"2024-04-05T03:02:50.002383+0000","last_unstale":"2024-04-05T03:02:50.471616+0000","last_undegraded":"2024-04-05T03:02:50.471616+0000","last_fullsized":"2024-04-05T03:02:50.471616+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:05:34.117134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470514+0000","last_change":"2024-04-05T03:02:48.457005+0000","last_active":"2024-04-05T03:02:50.470514+0000","last_peered":"2024-04-05T03:02:50.470514+0000","last_clean":"2024-04-05T03:02:50.470514+0000","last_became_active":"2024-04-05T03:02:48.456670+0000","last_became_peered":"2024-04-05T03:02:48.456670+0000","last_unstale":"2024-04-05T03:02:50.470514+0000","last_undegraded":"2024-04-05T03:02:50.470514+0000","last_fullsized":"2024-04-05T03:02:50.470514+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:45:12.114782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470894+0000","last_change":"2024-04-05T03:02:49.471218+0000","last_active":"2024-04-05T03:02:50.470894+0000","last_peered":"2024-04-05T03:02:50.470894+0000","last_clean":"2024-04-05T03:02:50.470894+0000","last_became_active":"2024-04-05T03:02:49.470918+0000","last_became_peered":"2024-04-05T03:02:49.470918+0000","last_unstale":"2024-04-05T03:02:50.470894+0000","last_undegraded":"2024-04-05T03:02:50.470894+0000","last_fullsized":"2024-04-05T03:02:50.470894+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:48:19.807934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.471027+0000","last_change":"2024-04-05T03:02:48.463560+0000","last_active":"2024-04-05T03:02:50.471027+0000","last_peered":"2024-04-05T03:02:50.471027+0000","last_clean":"2024-04-05T03:02:50.471027+0000","last_became_active":"2024-04-05T03:02:48.463245+0000","last_became_peered":"2024-04-05T03:02:48.463245+0000","last_unstale":"2024-04-05T03:02:50.471027+0000","last_undegraded":"2024-04-05T03:02:50.471027+0000","last_fullsized":"2024-04-05T03:02:50.471027+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:47:16.914625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-05T03:03:44.124416+0000","last_change":"2024-04-05T03:03:44.124631+0000","last_active":"2024-04-05T03:03:44.124380+0000","last_peered":"2024-04-05T03:02:52.144105+0000","last_clean":"2024-04-05T03:02:52.144105+0000","last_became_active":"2024-04-05T03:02:49.772945+0000","last_became_peered":"2024-04-05T03:02:49.772945+0000","last_unstale":"2024-04-05T03:03:44.124416+0000","last_undegraded":"2024-04-05T03:03:44.124416+0000","last_fullsized":"2024-04-05T03:03:44.124416+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:20:53.755431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470983+0000","last_change":"2024-04-05T03:02:48.457628+0000","last_active":"2024-04-05T03:02:50.470983+0000","last_peered":"2024-04-05T03:02:50.470983+0000","last_clean":"2024-04-05T03:02:50.470983+0000","last_became_active":"2024-04-05T03:02:48.457318+0000","last_became_peered":"2024-04-05T03:02:48.457318+0000","last_unstale":"2024-04-05T03:02:50.470983+0000","last_undegraded":"2024-04-05T03:02:50.470983+0000","last_fullsized":"2024-04-05T03:02:50.470983+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:32.231625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.470942+0000","last_change":"2024-04-05T03:02:50.004105+0000","last_active":"2024-04-05T03:02:50.470942+0000","last_peered":"2024-04-05T03:02:50.470942+0000","last_clean":"2024-04-05T03:02:50.470942+0000","last_became_active":"2024-04-05T03:02:50.003693+0000","last_became_peered":"2024-04-05T03:02:50.003693+0000","last_unstale":"2024-04-05T03:02:50.470942+0000","last_undegraded":"2024-04-05T03:02:50.470942+0000","last_fullsized":"2024-04-05T03:02:50.470942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:02.044564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:02:50.850887+0000","last_change":"2024-04-05T03:02:48.464554+0000","last_active":"2024-04-05T03:02:50.850887+0000","last_peered":"2024-04-05T03:02:50.850887+0000","last_clean":"2024-04-05T03:02:50.850887+0000","last_became_active":"2024-04-05T03:02:48.463923+0000","last_became_peered":"2024-04-05T03:02:48.463923+0000","last_unstale":"2024-04-05T03:02:50.850887+0000","last_undegraded":"2024-04-05T03:02:50.850887+0000","last_fullsized":"2024-04-05T03:02:50.850887+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:56.699626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-05T03:03:44.123283+0000","last_change":"2024-04-05T03:03:44.123504+0000","last_active":"2024-04-05T03:03:44.123251+0000","last_peered":"2024-04-05T03:02:50.471482+0000","last_clean":"2024-04-05T03:02:50.471482+0000","last_became_active":"2024-04-05T03:02:49.471004+0000","last_became_peered":"2024-04-05T03:02:49.471004+0000","last_unstale":"2024-04-05T03:03:44.123283+0000","last_undegraded":"2024-04-05T03:03:44.123283+0000","last_fullsized":"2024-04-05T03:03:44.123283+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:39:58.591265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4,5],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-05T03:03:28.862264+0000","last_change":"2024-04-05T03:02:48.456415+0000","last_active":"2024-04-05T03:03:28.862264+0000","last_peered":"2024-04-05T03:03:28.862264+0000","last_clean":"2024-04-05T03:03:28.862264+0000","last_became_active":"2024-04-05T03:02:48.455959+0000","last_became_peered":"2024-04-05T03:02:48.455959+0000","last_unstale":"2024-04-05T03:03:28.862264+0000","last_undegraded":"2024-04-05T03:03:28.862264+0000","last_fullsized":"2024-04-05T03:03:28.862264+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:52:33.266219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201314,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1323008,"data_stored":895341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.38400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51000000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.504}]},{"osd":1,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78700000000000003,"5min":0.878,"15min":0.878},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.379,"5min":0.371,"15min":0.371},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.48699999999999999}]},{"osd":2,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.41699999999999998}]},{"osd":3,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55400000000000005,"15min":0.55300000000000005},"min":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72999999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.44500000000000001}]},{"osd":4,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48199999999999998,"15min":0.48399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70099999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53300000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.502,"15min":0.50600000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.77400000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77100000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56799999999999995}]},{"osd":6,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.39800000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.432,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78800000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.55800000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462951,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298632,"kb_used_data":1276,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446520,"statfs":{"total":95995035648,"available":95689236480,"internally_reserved":0,"allocated":1306624,"data_stored":894951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35199999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.623},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.50800000000000001,"15min":0.51000000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.50600000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72599999999999998,"5min":0.88,"15min":0.88},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.51500000000000001,"15min":0.51900000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.58699999999999997}]},{"osd":3,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.40600000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76000000000000001,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.36499999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77700000000000002,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.47899999999999998}]},{"osd":4,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.42999999999999999,"15min":0.42699999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.432,"15min":0.41999999999999998},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.42899999999999999}]},{"osd":5,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.504,"15min":0.49199999999999999},"min":{"1min":0.36299999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.52600000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56399999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574911,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298036,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447116,"statfs":{"total":95995035648,"available":95689846784,"internally_reserved":0,"allocated":696320,"data_stored":302940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.312},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.374,"5min":0.379,"15min":0.34499999999999997},"min":{"1min":0.26100000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.54900000000000004,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.32600000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:03:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40600000000000003,"15min":0.39400000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.621,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.38},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.39200000000000002,"15min":0.37},"min":{"1min":0.24199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53800000000000003,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.42999999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.42099999999999999,"15min":0.42199999999999999},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69599999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.32400000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.9610000000000001,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.40699999999999997}]},{"osd":4,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63200000000000001,"15min":0.63900000000000001},"min":{"1min":0.45600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.85399999999999998,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86199999999999999,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.53800000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:29 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64100000000000001,"15min":0.64300000000000002},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93999999999999995,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":0.59599999999999997}]},{"osd":6,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.628,"15min":0.621},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.629,"15min":0.63600000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76500000000000001,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.505}]},{"osd":7,"last update":"Fri Apr 5 03:03:13 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59999999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.659,"5min":1.659,"15min":1.659},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.48999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738436,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298628,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446524,"statfs":{"total":95995035648,"available":95689240576,"internally_reserved":0,"allocated":1302528,"data_stored":894165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 03:02:56 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35399999999999998,"15min":0.32400000000000001},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":0.53700000000000003,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.374,"15min":0.34000000000000002},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.54200000000000004,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.33600000000000002}]},{"osd":2,"last update":"Fri Apr 5 03:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.46300000000000002,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.54000000000000004,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.376}]},{"osd":3,"last update":"Fri Apr 5 03:02:44 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.44900000000000001,"15min":0.44700000000000001},"min":{"1min":0.35499999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.59899999999999998,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.45300000000000001,"15min":0.45400000000000001},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59599999999999997,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.434}]},{"osd":4,"last update":"Fri Apr 5 03:03:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.875,"15min":0.875},"last":0.66000000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.41699999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79500000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83599999999999997,"5min":1.018,"15min":1.018},"last":0.61499999999999999}]},{"osd":6,"last update":"Fri Apr 5 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.42499999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81499999999999995,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.748},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64600000000000002,"15min":0.64500000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.64000000000000001}]},{"osd":7,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63500000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74099999999999999,"5min":0.875,"15min":0.875},"last":0.63},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.69699999999999995,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80600000000000005,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.83299999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":303809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51000000000000001,"5min":0.623,"15min":0.623},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39000000000000001,"15min":0.39400000000000002},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.53000000000000003,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.29299999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.39500000000000002,"15min":0.40500000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57599999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.54200000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39900000000000002}]},{"osd":3,"last update":"Fri Apr 5 03:02:44 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.45900000000000002,"15min":0.46400000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.70299999999999996,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.45000000000000001,"15min":0.45800000000000002},"min":{"1min":0.313,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.33900000000000002}]},{"osd":4,"last update":"Fri Apr 5 03:03:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.41299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.83699999999999997,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":12.170999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86799999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.56699999999999995}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80800000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.66600000000000004,"15min":0.60899999999999999},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":12.968999999999999,"5min":12.968999999999999,"15min":12.968999999999999},"last":0.57799999999999996}]},{"osd":6,"last update":"Fri Apr 5 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.83399999999999996,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.45400000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73599999999999999,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.61699999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:03:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.63900000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149749,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298044,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447108,"statfs":{"total":95995035648,"available":95689838592,"internally_reserved":0,"allocated":704512,"data_stored":303295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.39000000000000001,"15min":0.38900000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.439,"15min":0.44500000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.5740000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.34899999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.45900000000000002,"15min":0.45100000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.45600000000000002,"15min":0.45900000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.468,"5min":1.468,"15min":1.468},"last":0.32600000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:02:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.30399999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.5009999999999999,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.441}]},{"osd":4,"last update":"Fri Apr 5 03:03:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.434,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.96899999999999997,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.63100000000000001}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.432,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":1.002,"15min":1.002},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63600000000000001,"15min":0.64300000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.67500000000000004}]},{"osd":6,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.63900000000000001,"15min":0.64700000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.873,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89400000000000002,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.61699999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.63700000000000001,"15min":0.64100000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46800000000000003,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.65600000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986224,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":724992,"data_stored":304903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.56100000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54700000000000004,"15min":0.56000000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66700000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.47499999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.753,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.499},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55200000000000005,"15min":0.56200000000000006},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.67700000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.59399999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.64600000000000002,"15min":0.57299999999999995},"min":{"1min":0.372,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":11.239000000000001,"5min":11.239000000000001,"15min":11.239000000000001},"last":0.63},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57199999999999995,"15min":0.58999999999999997},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69499999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52000000000000002}]},{"osd":3,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.67900000000000005,"15min":0.61499999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.133,"5min":0.69099999999999995,"15min":0.625},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.221,"5min":11.221,"15min":11.221},"last":0.61899999999999999}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.40200000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.432},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.623,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.45100000000000001}]},{"osd":6,"last update":"Fri Apr 5 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.45500000000000002,"15min":0.44700000000000001},"min":{"1min":0.34399999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.376},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45700000000000002,"15min":0.44900000000000001},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66900000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.54800000000000004}]},{"osd":7,"last update":"Fri Apr 5 03:03:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.49299999999999999,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.58599999999999997,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.438,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.35899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.56999999999999995,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.436}]}]},{"osd":5,"up_from":39,"seq":167503724587,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":303845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74399999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.44600000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.39600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.78400000000000003,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.63700000000000001,"15min":0.58099999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.2070000000000007,"5min":9.2070000000000007,"15min":9.2070000000000007},"last":0.495}]},{"osd":2,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.443,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.55100000000000005,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.81599999999999995,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.70999999999999996}]},{"osd":3,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.75,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.56699999999999995,"15min":0.55700000000000005},"min":{"1min":0.437,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78100000000000003,"5min":1.329,"15min":1.329},"last":0.68300000000000005}]},{"osd":4,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.40899999999999997,"15min":0.38100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.42999999999999999,"15min":0.40200000000000002},"min":{"1min":0.32300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.71499999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.58199999999999996}]},{"osd":6,"last update":"Fri Apr 5 03:02:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.46800000000000003,"15min":0.45900000000000002},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.47999999999999998,"15min":0.46600000000000003},"min":{"1min":0.38,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.66400000000000003}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81999999999999995,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77000000000000002,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.622}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:03:50.189 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:03:50.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:50 smithi033 ceph-mon[40008]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:50.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:50 smithi033 ceph-mon[29244]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:50.999 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:51.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:50 smithi151 ceph-mon[32789]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:52.145 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:52.146 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.412 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:03:52 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:52] "GET /metrics HTTP/1.1" 200 239207 "" "Prometheus/2.43.0" 2024-04-05T03:03:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:52 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:52 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:52 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:52 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:52 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:03:52.886 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:03:52.887 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:03:53.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:53 smithi033 ceph-mon[40008]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:53.253 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:53 smithi033 ceph-mon[29244]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:53 smithi151 ceph-mon[32789]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:53.606 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":75,"stamp":"2024-04-05T03:03:52.058163+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386256,"kb_used_data":7396,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574960,"statfs":{"total":767960285184,"available":765516759040,"internally_reserved":0,"allocated":7573504,"data_stored":4259305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001581"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125656+0000","last_change":"2024-04-05T03:02:48.902639+0000","last_active":"2024-04-05T03:03:44.125656+0000","last_peered":"2024-04-05T03:03:44.125656+0000","last_clean":"2024-04-05T03:03:44.125656+0000","last_became_active":"2024-04-05T03:02:48.902301+0000","last_became_peered":"2024-04-05T03:02:48.902301+0000","last_unstale":"2024-04-05T03:03:44.125656+0000","last_undegraded":"2024-04-05T03:03:44.125656+0000","last_fullsized":"2024-04-05T03:03:44.125656+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:20:53.245725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122247+0000","last_change":"2024-04-05T03:02:48.457889+0000","last_active":"2024-04-05T03:03:44.122247+0000","last_peered":"2024-04-05T03:03:44.122247+0000","last_clean":"2024-04-05T03:03:44.122247+0000","last_became_active":"2024-04-05T03:02:48.457391+0000","last_became_peered":"2024-04-05T03:02:48.457391+0000","last_unstale":"2024-04-05T03:03:44.122247+0000","last_undegraded":"2024-04-05T03:03:44.122247+0000","last_fullsized":"2024-04-05T03:03:44.122247+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:38:13.647069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122541+0000","last_change":"2024-04-05T03:02:49.746985+0000","last_active":"2024-04-05T03:03:44.122541+0000","last_peered":"2024-04-05T03:03:44.122541+0000","last_clean":"2024-04-05T03:03:44.122541+0000","last_became_active":"2024-04-05T03:02:49.746610+0000","last_became_peered":"2024-04-05T03:02:49.746610+0000","last_unstale":"2024-04-05T03:03:44.122541+0000","last_undegraded":"2024-04-05T03:03:44.122541+0000","last_fullsized":"2024-04-05T03:03:44.122541+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:56:20.326917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122901+0000","last_change":"2024-04-05T03:02:48.459650+0000","last_active":"2024-04-05T03:03:44.122901+0000","last_peered":"2024-04-05T03:03:44.122901+0000","last_clean":"2024-04-05T03:03:44.122901+0000","last_became_active":"2024-04-05T03:02:48.459355+0000","last_became_peered":"2024-04-05T03:02:48.459355+0000","last_unstale":"2024-04-05T03:03:44.122901+0000","last_undegraded":"2024-04-05T03:03:44.122901+0000","last_fullsized":"2024-04-05T03:03:44.122901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:23:16.774935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125932+0000","last_change":"2024-04-05T03:02:48.644045+0000","last_active":"2024-04-05T03:03:44.125932+0000","last_peered":"2024-04-05T03:03:44.125932+0000","last_clean":"2024-04-05T03:03:44.125932+0000","last_became_active":"2024-04-05T03:02:48.643795+0000","last_became_peered":"2024-04-05T03:02:48.643795+0000","last_unstale":"2024-04-05T03:03:44.125932+0000","last_undegraded":"2024-04-05T03:03:44.125932+0000","last_fullsized":"2024-04-05T03:03:44.125932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:49:15.865276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122130+0000","last_change":"2024-04-05T03:02:48.458396+0000","last_active":"2024-04-05T03:03:44.122130+0000","last_peered":"2024-04-05T03:03:44.122130+0000","last_clean":"2024-04-05T03:03:44.122130+0000","last_became_active":"2024-04-05T03:02:48.458092+0000","last_became_peered":"2024-04-05T03:02:48.458092+0000","last_unstale":"2024-04-05T03:03:44.122130+0000","last_undegraded":"2024-04-05T03:03:44.122130+0000","last_fullsized":"2024-04-05T03:03:44.122130+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:43.973088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126011+0000","last_change":"2024-04-05T03:02:48.472069+0000","last_active":"2024-04-05T03:03:44.126011+0000","last_peered":"2024-04-05T03:03:44.126011+0000","last_clean":"2024-04-05T03:03:44.126011+0000","last_became_active":"2024-04-05T03:02:48.471739+0000","last_became_peered":"2024-04-05T03:02:48.471739+0000","last_unstale":"2024-04-05T03:03:44.126011+0000","last_undegraded":"2024-04-05T03:03:44.126011+0000","last_fullsized":"2024-04-05T03:03:44.126011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:44:49.173752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121975+0000","last_change":"2024-04-05T03:02:48.459793+0000","last_active":"2024-04-05T03:03:44.121975+0000","last_peered":"2024-04-05T03:03:44.121975+0000","last_clean":"2024-04-05T03:03:44.121975+0000","last_became_active":"2024-04-05T03:02:48.459518+0000","last_became_peered":"2024-04-05T03:02:48.459518+0000","last_unstale":"2024-04-05T03:03:44.121975+0000","last_undegraded":"2024-04-05T03:03:44.121975+0000","last_fullsized":"2024-04-05T03:03:44.121975+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:03:23.429419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127208+0000","last_change":"2024-04-05T03:02:48.473124+0000","last_active":"2024-04-05T03:03:44.127208+0000","last_peered":"2024-04-05T03:03:44.127208+0000","last_clean":"2024-04-05T03:03:44.127208+0000","last_became_active":"2024-04-05T03:02:48.472816+0000","last_became_peered":"2024-04-05T03:02:48.472816+0000","last_unstale":"2024-04-05T03:03:44.127208+0000","last_undegraded":"2024-04-05T03:03:44.127208+0000","last_fullsized":"2024-04-05T03:03:44.127208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:09:10.287592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121649+0000","last_change":"2024-04-05T03:02:48.457652+0000","last_active":"2024-04-05T03:03:44.121649+0000","last_peered":"2024-04-05T03:03:44.121649+0000","last_clean":"2024-04-05T03:03:44.121649+0000","last_became_active":"2024-04-05T03:02:48.457340+0000","last_became_peered":"2024-04-05T03:02:48.457340+0000","last_unstale":"2024-04-05T03:03:44.121649+0000","last_undegraded":"2024-04-05T03:03:44.121649+0000","last_fullsized":"2024-04-05T03:03:44.121649+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:04:12.013393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125917+0000","last_change":"2024-04-05T03:02:48.902933+0000","last_active":"2024-04-05T03:03:44.125917+0000","last_peered":"2024-04-05T03:03:44.125917+0000","last_clean":"2024-04-05T03:03:44.125917+0000","last_became_active":"2024-04-05T03:02:48.902583+0000","last_became_peered":"2024-04-05T03:02:48.902583+0000","last_unstale":"2024-04-05T03:03:44.125917+0000","last_undegraded":"2024-04-05T03:03:44.125917+0000","last_fullsized":"2024-04-05T03:03:44.125917+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:11:03.431547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121971+0000","last_change":"2024-04-05T03:02:48.465598+0000","last_active":"2024-04-05T03:03:44.121971+0000","last_peered":"2024-04-05T03:03:44.121971+0000","last_clean":"2024-04-05T03:03:44.121971+0000","last_became_active":"2024-04-05T03:02:48.465280+0000","last_became_peered":"2024-04-05T03:02:48.465280+0000","last_unstale":"2024-04-05T03:03:44.121971+0000","last_undegraded":"2024-04-05T03:03:44.121971+0000","last_fullsized":"2024-04-05T03:03:44.121971+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:27:18.537900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125053+0000","last_change":"2024-04-05T03:01:43.659234+0000","last_active":"2024-04-05T03:03:44.125053+0000","last_peered":"2024-04-05T03:03:44.125053+0000","last_clean":"2024-04-05T03:03:44.125053+0000","last_became_active":"2024-04-05T03:01:43.658801+0000","last_became_peered":"2024-04-05T03:01:43.658801+0000","last_unstale":"2024-04-05T03:03:44.125053+0000","last_undegraded":"2024-04-05T03:03:44.125053+0000","last_fullsized":"2024-04-05T03:03:44.125053+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:39:37.401263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122318+0000","last_change":"2024-04-05T03:02:50.003751+0000","last_active":"2024-04-05T03:03:44.122318+0000","last_peered":"2024-04-05T03:03:44.122318+0000","last_clean":"2024-04-05T03:03:44.122318+0000","last_became_active":"2024-04-05T03:02:50.003472+0000","last_became_peered":"2024-04-05T03:02:50.003472+0000","last_unstale":"2024-04-05T03:03:44.122318+0000","last_undegraded":"2024-04-05T03:03:44.122318+0000","last_fullsized":"2024-04-05T03:03:44.122318+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:46:41.938159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125410+0000","last_change":"2024-04-05T03:01:43.656498+0000","last_active":"2024-04-05T03:03:44.125410+0000","last_peered":"2024-04-05T03:03:44.125410+0000","last_clean":"2024-04-05T03:03:44.125410+0000","last_became_active":"2024-04-05T03:01:43.656159+0000","last_became_peered":"2024-04-05T03:01:43.656159+0000","last_unstale":"2024-04-05T03:03:44.125410+0000","last_undegraded":"2024-04-05T03:03:44.125410+0000","last_fullsized":"2024-04-05T03:03:44.125410+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:39.619074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123066+0000","last_change":"2024-04-05T03:02:48.458703+0000","last_active":"2024-04-05T03:03:44.123066+0000","last_peered":"2024-04-05T03:03:44.123066+0000","last_clean":"2024-04-05T03:03:44.123066+0000","last_became_active":"2024-04-05T03:02:48.457630+0000","last_became_peered":"2024-04-05T03:02:48.457630+0000","last_unstale":"2024-04-05T03:03:44.123066+0000","last_undegraded":"2024-04-05T03:03:44.123066+0000","last_fullsized":"2024-04-05T03:03:44.123066+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:42:21.323311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122148+0000","last_change":"2024-04-05T03:02:50.002775+0000","last_active":"2024-04-05T03:03:44.122148+0000","last_peered":"2024-04-05T03:03:44.122148+0000","last_clean":"2024-04-05T03:03:44.122148+0000","last_became_active":"2024-04-05T03:02:50.002356+0000","last_became_peered":"2024-04-05T03:02:50.002356+0000","last_unstale":"2024-04-05T03:03:44.122148+0000","last_undegraded":"2024-04-05T03:03:44.122148+0000","last_fullsized":"2024-04-05T03:03:44.122148+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:47:10.699733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125761+0000","last_change":"2024-04-05T03:02:48.456621+0000","last_active":"2024-04-05T03:03:44.125761+0000","last_peered":"2024-04-05T03:03:44.125761+0000","last_clean":"2024-04-05T03:03:44.125761+0000","last_became_active":"2024-04-05T03:02:48.455491+0000","last_became_peered":"2024-04-05T03:02:48.455491+0000","last_unstale":"2024-04-05T03:03:44.125761+0000","last_undegraded":"2024-04-05T03:03:44.125761+0000","last_fullsized":"2024-04-05T03:03:44.125761+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:38.283998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127688+0000","last_change":"2024-04-05T03:01:43.910231+0000","last_active":"2024-04-05T03:03:44.127688+0000","last_peered":"2024-04-05T03:03:44.127688+0000","last_clean":"2024-04-05T03:03:44.127688+0000","last_became_active":"2024-04-05T03:01:43.909886+0000","last_became_peered":"2024-04-05T03:01:43.909886+0000","last_unstale":"2024-04-05T03:03:44.127688+0000","last_undegraded":"2024-04-05T03:03:44.127688+0000","last_fullsized":"2024-04-05T03:03:44.127688+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:22:51.464507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122040+0000","last_change":"2024-04-05T03:02:49.746307+0000","last_active":"2024-04-05T03:03:44.122040+0000","last_peered":"2024-04-05T03:03:44.122040+0000","last_clean":"2024-04-05T03:03:44.122040+0000","last_became_active":"2024-04-05T03:02:49.745976+0000","last_became_peered":"2024-04-05T03:02:49.745976+0000","last_unstale":"2024-04-05T03:03:44.122040+0000","last_undegraded":"2024-04-05T03:03:44.122040+0000","last_fullsized":"2024-04-05T03:03:44.122040+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:49:15.142317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121543+0000","last_change":"2024-04-05T03:02:48.461046+0000","last_active":"2024-04-05T03:03:44.121543+0000","last_peered":"2024-04-05T03:03:44.121543+0000","last_clean":"2024-04-05T03:03:44.121543+0000","last_became_active":"2024-04-05T03:02:48.460678+0000","last_became_peered":"2024-04-05T03:02:48.460678+0000","last_unstale":"2024-04-05T03:03:44.121543+0000","last_undegraded":"2024-04-05T03:03:44.121543+0000","last_fullsized":"2024-04-05T03:03:44.121543+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:23.864420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126652+0000","last_change":"2024-04-05T03:01:43.910268+0000","last_active":"2024-04-05T03:03:44.126652+0000","last_peered":"2024-04-05T03:03:44.126652+0000","last_clean":"2024-04-05T03:03:44.126652+0000","last_became_active":"2024-04-05T03:01:43.909991+0000","last_became_peered":"2024-04-05T03:01:43.909991+0000","last_unstale":"2024-04-05T03:03:44.126652+0000","last_undegraded":"2024-04-05T03:03:44.126652+0000","last_fullsized":"2024-04-05T03:03:44.126652+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:12.008125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124460+0000","last_change":"2024-04-05T03:02:50.003073+0000","last_active":"2024-04-05T03:03:44.124460+0000","last_peered":"2024-04-05T03:03:44.124460+0000","last_clean":"2024-04-05T03:03:44.124460+0000","last_became_active":"2024-04-05T03:02:50.002726+0000","last_became_peered":"2024-04-05T03:02:50.002726+0000","last_unstale":"2024-04-05T03:03:44.124460+0000","last_undegraded":"2024-04-05T03:03:44.124460+0000","last_fullsized":"2024-04-05T03:03:44.124460+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:50:50.549731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122646+0000","last_change":"2024-04-05T03:02:48.460577+0000","last_active":"2024-04-05T03:03:44.122646+0000","last_peered":"2024-04-05T03:03:44.122646+0000","last_clean":"2024-04-05T03:03:44.122646+0000","last_became_active":"2024-04-05T03:02:48.460216+0000","last_became_peered":"2024-04-05T03:02:48.460216+0000","last_unstale":"2024-04-05T03:03:44.122646+0000","last_undegraded":"2024-04-05T03:03:44.122646+0000","last_fullsized":"2024-04-05T03:03:44.122646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:35:58.345475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126636+0000","last_change":"2024-04-05T03:01:14.837104+0000","last_active":"2024-04-05T03:03:44.126636+0000","last_peered":"2024-04-05T03:03:44.126636+0000","last_clean":"2024-04-05T03:03:44.126636+0000","last_became_active":"2024-04-05T03:01:14.836690+0000","last_became_peered":"2024-04-05T03:01:14.836690+0000","last_unstale":"2024-04-05T03:03:44.126636+0000","last_undegraded":"2024-04-05T03:03:44.126636+0000","last_fullsized":"2024-04-05T03:03:44.126636+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:48:28.544381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121869+0000","last_change":"2024-04-05T03:01:43.659091+0000","last_active":"2024-04-05T03:03:44.121869+0000","last_peered":"2024-04-05T03:03:44.121869+0000","last_clean":"2024-04-05T03:03:44.121869+0000","last_became_active":"2024-04-05T03:01:43.658783+0000","last_became_peered":"2024-04-05T03:01:43.658783+0000","last_unstale":"2024-04-05T03:03:44.121869+0000","last_undegraded":"2024-04-05T03:03:44.121869+0000","last_fullsized":"2024-04-05T03:03:44.121869+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:34:20.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125743+0000","last_change":"2024-04-05T03:02:49.773210+0000","last_active":"2024-04-05T03:03:44.125743+0000","last_peered":"2024-04-05T03:03:44.125743+0000","last_clean":"2024-04-05T03:03:44.125743+0000","last_became_active":"2024-04-05T03:02:49.772812+0000","last_became_peered":"2024-04-05T03:02:49.772812+0000","last_unstale":"2024-04-05T03:03:44.125743+0000","last_undegraded":"2024-04-05T03:03:44.125743+0000","last_fullsized":"2024-04-05T03:03:44.125743+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:29:41.425482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125536+0000","last_change":"2024-04-05T03:02:48.457064+0000","last_active":"2024-04-05T03:03:44.125536+0000","last_peered":"2024-04-05T03:03:44.125536+0000","last_clean":"2024-04-05T03:03:44.125536+0000","last_became_active":"2024-04-05T03:02:48.456787+0000","last_became_peered":"2024-04-05T03:02:48.456787+0000","last_unstale":"2024-04-05T03:03:44.125536+0000","last_undegraded":"2024-04-05T03:03:44.125536+0000","last_fullsized":"2024-04-05T03:03:44.125536+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:01:04.753701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126939+0000","last_change":"2024-04-05T03:01:43.910452+0000","last_active":"2024-04-05T03:03:44.126939+0000","last_peered":"2024-04-05T03:03:44.126939+0000","last_clean":"2024-04-05T03:03:44.126939+0000","last_became_active":"2024-04-05T03:01:43.909876+0000","last_became_peered":"2024-04-05T03:01:43.909876+0000","last_unstale":"2024-04-05T03:03:44.126939+0000","last_undegraded":"2024-04-05T03:03:44.126939+0000","last_fullsized":"2024-04-05T03:03:44.126939+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:07:36.880582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125680+0000","last_change":"2024-04-05T03:02:49.746957+0000","last_active":"2024-04-05T03:03:44.125680+0000","last_peered":"2024-04-05T03:03:44.125680+0000","last_clean":"2024-04-05T03:03:44.125680+0000","last_became_active":"2024-04-05T03:02:49.746064+0000","last_became_peered":"2024-04-05T03:02:49.746064+0000","last_unstale":"2024-04-05T03:03:44.125680+0000","last_undegraded":"2024-04-05T03:03:44.125680+0000","last_fullsized":"2024-04-05T03:03:44.125680+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:42:42.677796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126016+0000","last_change":"2024-04-05T03:02:48.455210+0000","last_active":"2024-04-05T03:03:44.126016+0000","last_peered":"2024-04-05T03:03:44.126016+0000","last_clean":"2024-04-05T03:03:44.126016+0000","last_became_active":"2024-04-05T03:02:48.453370+0000","last_became_peered":"2024-04-05T03:02:48.453370+0000","last_unstale":"2024-04-05T03:03:44.126016+0000","last_undegraded":"2024-04-05T03:03:44.126016+0000","last_fullsized":"2024-04-05T03:03:44.126016+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:50.364653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121829+0000","last_change":"2024-04-05T03:01:43.656346+0000","last_active":"2024-04-05T03:03:44.121829+0000","last_peered":"2024-04-05T03:03:44.121829+0000","last_clean":"2024-04-05T03:03:44.121829+0000","last_became_active":"2024-04-05T03:01:43.656045+0000","last_became_peered":"2024-04-05T03:01:43.656045+0000","last_unstale":"2024-04-05T03:03:44.121829+0000","last_undegraded":"2024-04-05T03:03:44.121829+0000","last_fullsized":"2024-04-05T03:03:44.121829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:40:55.612040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125054+0000","last_change":"2024-04-05T03:02:49.772316+0000","last_active":"2024-04-05T03:03:44.125054+0000","last_peered":"2024-04-05T03:03:44.125054+0000","last_clean":"2024-04-05T03:03:44.125054+0000","last_became_active":"2024-04-05T03:02:49.772060+0000","last_became_peered":"2024-04-05T03:02:49.772060+0000","last_unstale":"2024-04-05T03:03:44.125054+0000","last_undegraded":"2024-04-05T03:03:44.125054+0000","last_fullsized":"2024-04-05T03:03:44.125054+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:05:14.265331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122117+0000","last_change":"2024-04-05T03:02:48.457200+0000","last_active":"2024-04-05T03:03:44.122117+0000","last_peered":"2024-04-05T03:03:44.122117+0000","last_clean":"2024-04-05T03:03:44.122117+0000","last_became_active":"2024-04-05T03:02:48.456346+0000","last_became_peered":"2024-04-05T03:02:48.456346+0000","last_unstale":"2024-04-05T03:03:44.122117+0000","last_undegraded":"2024-04-05T03:03:44.122117+0000","last_fullsized":"2024-04-05T03:03:44.122117+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:22.908070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125596+0000","last_change":"2024-04-05T03:01:43.910340+0000","last_active":"2024-04-05T03:03:44.125596+0000","last_peered":"2024-04-05T03:03:44.125596+0000","last_clean":"2024-04-05T03:03:44.125596+0000","last_became_active":"2024-04-05T03:01:43.909987+0000","last_became_peered":"2024-04-05T03:01:43.909987+0000","last_unstale":"2024-04-05T03:03:44.125596+0000","last_undegraded":"2024-04-05T03:03:44.125596+0000","last_fullsized":"2024-04-05T03:03:44.125596+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:42.918552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125820+0000","last_change":"2024-04-05T03:02:49.469399+0000","last_active":"2024-04-05T03:03:44.125820+0000","last_peered":"2024-04-05T03:03:44.125820+0000","last_clean":"2024-04-05T03:03:44.125820+0000","last_became_active":"2024-04-05T03:02:49.469112+0000","last_became_peered":"2024-04-05T03:02:49.469112+0000","last_unstale":"2024-04-05T03:03:44.125820+0000","last_undegraded":"2024-04-05T03:03:44.125820+0000","last_fullsized":"2024-04-05T03:03:44.125820+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:59:24.415770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122553+0000","last_change":"2024-04-05T03:02:48.460772+0000","last_active":"2024-04-05T03:03:44.122553+0000","last_peered":"2024-04-05T03:03:44.122553+0000","last_clean":"2024-04-05T03:03:44.122553+0000","last_became_active":"2024-04-05T03:02:48.458509+0000","last_became_peered":"2024-04-05T03:02:48.458509+0000","last_unstale":"2024-04-05T03:03:44.122553+0000","last_undegraded":"2024-04-05T03:03:44.122553+0000","last_fullsized":"2024-04-05T03:03:44.122553+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:56:41.129273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126900+0000","last_change":"2024-04-05T03:02:48.902626+0000","last_active":"2024-04-05T03:03:44.126900+0000","last_peered":"2024-04-05T03:03:44.126900+0000","last_clean":"2024-04-05T03:03:44.126900+0000","last_became_active":"2024-04-05T03:02:48.902271+0000","last_became_peered":"2024-04-05T03:02:48.902271+0000","last_unstale":"2024-04-05T03:03:44.126900+0000","last_undegraded":"2024-04-05T03:03:44.126900+0000","last_fullsized":"2024-04-05T03:03:44.126900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:31:05.812000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125944+0000","last_change":"2024-04-05T03:02:48.448680+0000","last_active":"2024-04-05T03:03:44.125944+0000","last_peered":"2024-04-05T03:03:44.125944+0000","last_clean":"2024-04-05T03:03:44.125944+0000","last_became_active":"2024-04-05T03:02:48.448318+0000","last_became_peered":"2024-04-05T03:02:48.448318+0000","last_unstale":"2024-04-05T03:03:44.125944+0000","last_undegraded":"2024-04-05T03:03:44.125944+0000","last_fullsized":"2024-04-05T03:03:44.125944+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:17.475717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126861+0000","last_change":"2024-04-05T03:02:48.472192+0000","last_active":"2024-04-05T03:03:44.126861+0000","last_peered":"2024-04-05T03:03:44.126861+0000","last_clean":"2024-04-05T03:03:44.126861+0000","last_became_active":"2024-04-05T03:02:48.471904+0000","last_became_peered":"2024-04-05T03:02:48.471904+0000","last_unstale":"2024-04-05T03:03:44.126861+0000","last_undegraded":"2024-04-05T03:03:44.126861+0000","last_fullsized":"2024-04-05T03:03:44.126861+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:38:38.241850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124325+0000","last_change":"2024-04-05T03:02:48.459300+0000","last_active":"2024-04-05T03:03:44.124325+0000","last_peered":"2024-04-05T03:03:44.124325+0000","last_clean":"2024-04-05T03:03:44.124325+0000","last_became_active":"2024-04-05T03:02:48.458886+0000","last_became_peered":"2024-04-05T03:02:48.458886+0000","last_unstale":"2024-04-05T03:03:44.124325+0000","last_undegraded":"2024-04-05T03:03:44.124325+0000","last_fullsized":"2024-04-05T03:03:44.124325+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:55:07.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126488+0000","last_change":"2024-04-05T03:02:48.904058+0000","last_active":"2024-04-05T03:03:44.126488+0000","last_peered":"2024-04-05T03:03:44.126488+0000","last_clean":"2024-04-05T03:03:44.126488+0000","last_became_active":"2024-04-05T03:02:48.903756+0000","last_became_peered":"2024-04-05T03:02:48.903756+0000","last_unstale":"2024-04-05T03:03:44.126488+0000","last_undegraded":"2024-04-05T03:03:44.126488+0000","last_fullsized":"2024-04-05T03:03:44.126488+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:11:54.123968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123183+0000","last_change":"2024-04-05T03:02:48.457251+0000","last_active":"2024-04-05T03:03:44.123183+0000","last_peered":"2024-04-05T03:03:44.123183+0000","last_clean":"2024-04-05T03:03:44.123183+0000","last_became_active":"2024-04-05T03:02:48.456950+0000","last_became_peered":"2024-04-05T03:02:48.456950+0000","last_unstale":"2024-04-05T03:03:44.123183+0000","last_undegraded":"2024-04-05T03:03:44.123183+0000","last_fullsized":"2024-04-05T03:03:44.123183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:56:54.797317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121818+0000","last_change":"2024-04-05T03:02:49.473606+0000","last_active":"2024-04-05T03:03:44.121818+0000","last_peered":"2024-04-05T03:03:44.121818+0000","last_clean":"2024-04-05T03:03:44.121818+0000","last_became_active":"2024-04-05T03:02:49.473288+0000","last_became_peered":"2024-04-05T03:02:49.473288+0000","last_unstale":"2024-04-05T03:03:44.121818+0000","last_undegraded":"2024-04-05T03:03:44.121818+0000","last_fullsized":"2024-04-05T03:03:44.121818+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:16:58.265293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121870+0000","last_change":"2024-04-05T03:02:48.461876+0000","last_active":"2024-04-05T03:03:44.121870+0000","last_peered":"2024-04-05T03:03:44.121870+0000","last_clean":"2024-04-05T03:03:44.121870+0000","last_became_active":"2024-04-05T03:02:48.461576+0000","last_became_peered":"2024-04-05T03:02:48.461576+0000","last_unstale":"2024-04-05T03:03:44.121870+0000","last_undegraded":"2024-04-05T03:03:44.121870+0000","last_fullsized":"2024-04-05T03:03:44.121870+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:24:53.545330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.159681+0000","last_change":"2024-04-05T03:03:45.159681+0000","last_active":"2024-04-05T03:03:45.159681+0000","last_peered":"2024-04-05T03:03:45.159681+0000","last_clean":"2024-04-05T03:03:45.159681+0000","last_became_active":"2024-04-05T03:03:45.159381+0000","last_became_peered":"2024-04-05T03:03:45.159381+0000","last_unstale":"2024-04-05T03:03:45.159681+0000","last_undegraded":"2024-04-05T03:03:45.159681+0000","last_fullsized":"2024-04-05T03:03:45.159681+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:47:51.454737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127784+0000","last_change":"2024-04-05T03:02:48.461252+0000","last_active":"2024-04-05T03:03:44.127784+0000","last_peered":"2024-04-05T03:03:44.127784+0000","last_clean":"2024-04-05T03:03:44.127784+0000","last_became_active":"2024-04-05T03:02:48.460982+0000","last_became_peered":"2024-04-05T03:02:48.460982+0000","last_unstale":"2024-04-05T03:03:44.127784+0000","last_undegraded":"2024-04-05T03:03:44.127784+0000","last_fullsized":"2024-04-05T03:03:44.127784+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:03.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125800+0000","last_change":"2024-04-05T03:02:48.472006+0000","last_active":"2024-04-05T03:03:44.125800+0000","last_peered":"2024-04-05T03:03:44.125800+0000","last_clean":"2024-04-05T03:03:44.125800+0000","last_became_active":"2024-04-05T03:02:48.471696+0000","last_became_peered":"2024-04-05T03:02:48.471696+0000","last_unstale":"2024-04-05T03:03:44.125800+0000","last_undegraded":"2024-04-05T03:03:44.125800+0000","last_fullsized":"2024-04-05T03:03:44.125800+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:31:37.135819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127543+0000","last_change":"2024-04-05T03:02:48.457001+0000","last_active":"2024-04-05T03:03:44.127543+0000","last_peered":"2024-04-05T03:03:44.127543+0000","last_clean":"2024-04-05T03:03:44.127543+0000","last_became_active":"2024-04-05T03:02:48.456334+0000","last_became_peered":"2024-04-05T03:02:48.456334+0000","last_unstale":"2024-04-05T03:03:44.127543+0000","last_undegraded":"2024-04-05T03:03:44.127543+0000","last_fullsized":"2024-04-05T03:03:44.127543+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:16:57.078617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121741+0000","last_change":"2024-04-05T03:02:48.903134+0000","last_active":"2024-04-05T03:03:44.121741+0000","last_peered":"2024-04-05T03:03:44.121741+0000","last_clean":"2024-04-05T03:03:44.121741+0000","last_became_active":"2024-04-05T03:02:48.902836+0000","last_became_peered":"2024-04-05T03:02:48.902836+0000","last_unstale":"2024-04-05T03:03:44.121741+0000","last_undegraded":"2024-04-05T03:03:44.121741+0000","last_fullsized":"2024-04-05T03:03:44.121741+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:08:54.250101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127935+0000","last_change":"2024-04-05T03:02:48.457358+0000","last_active":"2024-04-05T03:03:44.127935+0000","last_peered":"2024-04-05T03:03:44.127935+0000","last_clean":"2024-04-05T03:03:44.127935+0000","last_became_active":"2024-04-05T03:02:48.456987+0000","last_became_peered":"2024-04-05T03:02:48.456987+0000","last_unstale":"2024-04-05T03:03:44.127935+0000","last_undegraded":"2024-04-05T03:03:44.127935+0000","last_fullsized":"2024-04-05T03:03:44.127935+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:22.057798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123071+0000","last_change":"2024-04-05T03:02:50.003001+0000","last_active":"2024-04-05T03:03:44.123071+0000","last_peered":"2024-04-05T03:03:44.123071+0000","last_clean":"2024-04-05T03:03:44.123071+0000","last_became_active":"2024-04-05T03:02:50.002734+0000","last_became_peered":"2024-04-05T03:02:50.002734+0000","last_unstale":"2024-04-05T03:03:44.123071+0000","last_undegraded":"2024-04-05T03:03:44.123071+0000","last_fullsized":"2024-04-05T03:03:44.123071+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:52:55.958187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123875+0000","last_change":"2024-04-05T03:02:48.457497+0000","last_active":"2024-04-05T03:03:44.123875+0000","last_peered":"2024-04-05T03:03:44.123875+0000","last_clean":"2024-04-05T03:03:44.123875+0000","last_became_active":"2024-04-05T03:02:48.457161+0000","last_became_peered":"2024-04-05T03:02:48.457161+0000","last_unstale":"2024-04-05T03:03:44.123875+0000","last_undegraded":"2024-04-05T03:03:44.123875+0000","last_fullsized":"2024-04-05T03:03:44.123875+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:31:49.521313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121819+0000","last_change":"2024-04-05T03:02:50.003556+0000","last_active":"2024-04-05T03:03:44.121819+0000","last_peered":"2024-04-05T03:03:44.121819+0000","last_clean":"2024-04-05T03:03:44.121819+0000","last_became_active":"2024-04-05T03:02:50.003271+0000","last_became_peered":"2024-04-05T03:02:50.003271+0000","last_unstale":"2024-04-05T03:03:44.121819+0000","last_undegraded":"2024-04-05T03:03:44.121819+0000","last_fullsized":"2024-04-05T03:03:44.121819+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:00:59.126639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126848+0000","last_change":"2024-04-05T03:02:48.460853+0000","last_active":"2024-04-05T03:03:44.126848+0000","last_peered":"2024-04-05T03:03:44.126848+0000","last_clean":"2024-04-05T03:03:44.126848+0000","last_became_active":"2024-04-05T03:02:48.460557+0000","last_became_peered":"2024-04-05T03:02:48.460557+0000","last_unstale":"2024-04-05T03:03:44.126848+0000","last_undegraded":"2024-04-05T03:03:44.126848+0000","last_fullsized":"2024-04-05T03:03:44.126848+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:57.261993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122011+0000","last_change":"2024-04-05T03:02:48.902752+0000","last_active":"2024-04-05T03:03:44.122011+0000","last_peered":"2024-04-05T03:03:44.122011+0000","last_clean":"2024-04-05T03:03:44.122011+0000","last_became_active":"2024-04-05T03:02:48.902396+0000","last_became_peered":"2024-04-05T03:02:48.902396+0000","last_unstale":"2024-04-05T03:03:44.122011+0000","last_undegraded":"2024-04-05T03:03:44.122011+0000","last_fullsized":"2024-04-05T03:03:44.122011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:45:00.227236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121540+0000","last_change":"2024-04-05T03:02:48.461320+0000","last_active":"2024-04-05T03:03:44.121540+0000","last_peered":"2024-04-05T03:03:44.121540+0000","last_clean":"2024-04-05T03:03:44.121540+0000","last_became_active":"2024-04-05T03:02:48.461034+0000","last_became_peered":"2024-04-05T03:02:48.461034+0000","last_unstale":"2024-04-05T03:03:44.121540+0000","last_undegraded":"2024-04-05T03:03:44.121540+0000","last_fullsized":"2024-04-05T03:03:44.121540+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:34:08.199664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122756+0000","last_change":"2024-04-05T03:02:50.002729+0000","last_active":"2024-04-05T03:03:44.122756+0000","last_peered":"2024-04-05T03:03:44.122756+0000","last_clean":"2024-04-05T03:03:44.122756+0000","last_became_active":"2024-04-05T03:02:50.002342+0000","last_became_peered":"2024-04-05T03:02:50.002342+0000","last_unstale":"2024-04-05T03:03:44.122756+0000","last_undegraded":"2024-04-05T03:03:44.122756+0000","last_fullsized":"2024-04-05T03:03:44.122756+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:51:56.223511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127209+0000","last_change":"2024-04-05T03:02:48.459896+0000","last_active":"2024-04-05T03:03:44.127209+0000","last_peered":"2024-04-05T03:03:44.127209+0000","last_clean":"2024-04-05T03:03:44.127209+0000","last_became_active":"2024-04-05T03:02:48.458108+0000","last_became_peered":"2024-04-05T03:02:48.458108+0000","last_unstale":"2024-04-05T03:03:44.127209+0000","last_undegraded":"2024-04-05T03:03:44.127209+0000","last_fullsized":"2024-04-05T03:03:44.127209+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:41:12.902400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126553+0000","last_change":"2024-04-05T03:02:48.903143+0000","last_active":"2024-04-05T03:03:44.126553+0000","last_peered":"2024-04-05T03:03:44.126553+0000","last_clean":"2024-04-05T03:03:44.126553+0000","last_became_active":"2024-04-05T03:02:48.902265+0000","last_became_peered":"2024-04-05T03:02:48.902265+0000","last_unstale":"2024-04-05T03:03:44.126553+0000","last_undegraded":"2024-04-05T03:03:44.126553+0000","last_fullsized":"2024-04-05T03:03:44.126553+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:05:56.179576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126604+0000","last_change":"2024-04-05T03:02:48.455833+0000","last_active":"2024-04-05T03:03:44.126604+0000","last_peered":"2024-04-05T03:03:44.126604+0000","last_clean":"2024-04-05T03:03:44.126604+0000","last_became_active":"2024-04-05T03:02:48.455185+0000","last_became_peered":"2024-04-05T03:02:48.455185+0000","last_unstale":"2024-04-05T03:03:44.126604+0000","last_undegraded":"2024-04-05T03:03:44.126604+0000","last_fullsized":"2024-04-05T03:03:44.126604+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:02.621579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124971+0000","last_change":"2024-04-05T03:02:49.746606+0000","last_active":"2024-04-05T03:03:44.124971+0000","last_peered":"2024-04-05T03:03:44.124971+0000","last_clean":"2024-04-05T03:03:44.124971+0000","last_became_active":"2024-04-05T03:02:49.746308+0000","last_became_peered":"2024-04-05T03:02:49.746308+0000","last_unstale":"2024-04-05T03:03:44.124971+0000","last_undegraded":"2024-04-05T03:03:44.124971+0000","last_fullsized":"2024-04-05T03:03:44.124971+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:07:39.965218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127576+0000","last_change":"2024-04-05T03:02:48.456697+0000","last_active":"2024-04-05T03:03:44.127576+0000","last_peered":"2024-04-05T03:03:44.127576+0000","last_clean":"2024-04-05T03:03:44.127576+0000","last_became_active":"2024-04-05T03:02:48.456140+0000","last_became_peered":"2024-04-05T03:02:48.456140+0000","last_unstale":"2024-04-05T03:03:44.127576+0000","last_undegraded":"2024-04-05T03:03:44.127576+0000","last_fullsized":"2024-04-05T03:03:44.127576+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:57:31.290707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122585+0000","last_change":"2024-04-05T03:02:50.002658+0000","last_active":"2024-04-05T03:03:44.122585+0000","last_peered":"2024-04-05T03:03:44.122585+0000","last_clean":"2024-04-05T03:03:44.122585+0000","last_became_active":"2024-04-05T03:02:50.002383+0000","last_became_peered":"2024-04-05T03:02:50.002383+0000","last_unstale":"2024-04-05T03:03:44.122585+0000","last_undegraded":"2024-04-05T03:03:44.122585+0000","last_fullsized":"2024-04-05T03:03:44.122585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:05:34.117134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121968+0000","last_change":"2024-04-05T03:02:48.457005+0000","last_active":"2024-04-05T03:03:44.121968+0000","last_peered":"2024-04-05T03:03:44.121968+0000","last_clean":"2024-04-05T03:03:44.121968+0000","last_became_active":"2024-04-05T03:02:48.456670+0000","last_became_peered":"2024-04-05T03:02:48.456670+0000","last_unstale":"2024-04-05T03:03:44.121968+0000","last_undegraded":"2024-04-05T03:03:44.121968+0000","last_fullsized":"2024-04-05T03:03:44.121968+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:45:12.114782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122420+0000","last_change":"2024-04-05T03:02:49.471218+0000","last_active":"2024-04-05T03:03:44.122420+0000","last_peered":"2024-04-05T03:03:44.122420+0000","last_clean":"2024-04-05T03:03:44.122420+0000","last_became_active":"2024-04-05T03:02:49.470918+0000","last_became_peered":"2024-04-05T03:02:49.470918+0000","last_unstale":"2024-04-05T03:03:44.122420+0000","last_undegraded":"2024-04-05T03:03:44.122420+0000","last_fullsized":"2024-04-05T03:03:44.122420+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:48:19.807934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122364+0000","last_change":"2024-04-05T03:02:48.463560+0000","last_active":"2024-04-05T03:03:44.122364+0000","last_peered":"2024-04-05T03:03:44.122364+0000","last_clean":"2024-04-05T03:03:44.122364+0000","last_became_active":"2024-04-05T03:02:48.463245+0000","last_became_peered":"2024-04-05T03:02:48.463245+0000","last_unstale":"2024-04-05T03:03:44.122364+0000","last_undegraded":"2024-04-05T03:03:44.122364+0000","last_fullsized":"2024-04-05T03:03:44.122364+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:47:16.914625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.134193+0000","last_change":"2024-04-05T03:03:45.134193+0000","last_active":"2024-04-05T03:03:45.134193+0000","last_peered":"2024-04-05T03:03:45.134193+0000","last_clean":"2024-04-05T03:03:45.134193+0000","last_became_active":"2024-04-05T03:03:45.133891+0000","last_became_peered":"2024-04-05T03:03:45.133891+0000","last_unstale":"2024-04-05T03:03:45.134193+0000","last_undegraded":"2024-04-05T03:03:45.134193+0000","last_fullsized":"2024-04-05T03:03:45.134193+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:56:06.527860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122675+0000","last_change":"2024-04-05T03:02:48.457628+0000","last_active":"2024-04-05T03:03:44.122675+0000","last_peered":"2024-04-05T03:03:44.122675+0000","last_clean":"2024-04-05T03:03:44.122675+0000","last_became_active":"2024-04-05T03:02:48.457318+0000","last_became_peered":"2024-04-05T03:02:48.457318+0000","last_unstale":"2024-04-05T03:03:44.122675+0000","last_undegraded":"2024-04-05T03:03:44.122675+0000","last_fullsized":"2024-04-05T03:03:44.122675+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:32.231625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121893+0000","last_change":"2024-04-05T03:02:50.004105+0000","last_active":"2024-04-05T03:03:44.121893+0000","last_peered":"2024-04-05T03:03:44.121893+0000","last_clean":"2024-04-05T03:03:44.121893+0000","last_became_active":"2024-04-05T03:02:50.003693+0000","last_became_peered":"2024-04-05T03:02:50.003693+0000","last_unstale":"2024-04-05T03:03:44.121893+0000","last_undegraded":"2024-04-05T03:03:44.121893+0000","last_fullsized":"2024-04-05T03:03:44.121893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:02.044564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125825+0000","last_change":"2024-04-05T03:02:48.464554+0000","last_active":"2024-04-05T03:03:44.125825+0000","last_peered":"2024-04-05T03:03:44.125825+0000","last_clean":"2024-04-05T03:03:44.125825+0000","last_became_active":"2024-04-05T03:02:48.463923+0000","last_became_peered":"2024-04-05T03:02:48.463923+0000","last_unstale":"2024-04-05T03:03:44.125825+0000","last_undegraded":"2024-04-05T03:03:44.125825+0000","last_fullsized":"2024-04-05T03:03:44.125825+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:56.699626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.256204+0000","last_change":"2024-04-05T03:03:45.256204+0000","last_active":"2024-04-05T03:03:45.256204+0000","last_peered":"2024-04-05T03:03:45.256204+0000","last_clean":"2024-04-05T03:03:45.256204+0000","last_became_active":"2024-04-05T03:03:45.255812+0000","last_became_peered":"2024-04-05T03:03:45.255812+0000","last_unstale":"2024-04-05T03:03:45.256204+0000","last_undegraded":"2024-04-05T03:03:45.256204+0000","last_fullsized":"2024-04-05T03:03:45.256204+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:15:31.145449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125800+0000","last_change":"2024-04-05T03:02:48.456415+0000","last_active":"2024-04-05T03:03:44.125800+0000","last_peered":"2024-04-05T03:03:44.125800+0000","last_clean":"2024-04-05T03:03:44.125800+0000","last_became_active":"2024-04-05T03:02:48.455959+0000","last_became_peered":"2024-04-05T03:02:48.455959+0000","last_unstale":"2024-04-05T03:03:44.125800+0000","last_undegraded":"2024-04-05T03:03:44.125800+0000","last_fullsized":"2024-04-05T03:03:44.125800+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:52:33.266219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201315,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298660,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446492,"statfs":{"total":95995035648,"available":95689207808,"internally_reserved":0,"allocated":1335296,"data_stored":902348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.38400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51000000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71799999999999997}]},{"osd":1,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78700000000000003,"5min":0.878,"15min":0.878},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.379,"5min":0.371,"15min":0.371},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.629}]},{"osd":2,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.59299999999999997}]},{"osd":3,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55400000000000005,"15min":0.55300000000000005},"min":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72999999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.64700000000000002}]},{"osd":4,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48199999999999998,"15min":0.48399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70099999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.496}]},{"osd":5,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.502,"15min":0.50600000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.77400000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77100000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.47699999999999998}]},{"osd":6,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.39800000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.432,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78800000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.55000000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462952,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":901958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.71099999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.70899999999999996,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.55800000000000005}]},{"osd":1,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.39200000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.58499999999999996}]},{"osd":2,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.46200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69399999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.64900000000000002,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.48399999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57299999999999995,"15min":0.56000000000000005},"min":{"1min":0.438,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.42799999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.44,"15min":0.42999999999999999},"min":{"1min":0.315,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.45100000000000001,"15min":0.42699999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.51100000000000001}]},{"osd":5,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.46400000000000002,"15min":0.44700000000000001},"min":{"1min":0.36499999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.626,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.622},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.52600000000000002,"15min":0.499},"min":{"1min":0.40400000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69199999999999995}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.71899999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":708608,"data_stored":309947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.312},"min":{"1min":0.20399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.374,"5min":0.379,"15min":0.34499999999999997},"min":{"1min":0.26100000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.54900000000000004,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.51500000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:03:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40600000000000003,"15min":0.39400000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.621,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.39200000000000002,"15min":0.37},"min":{"1min":0.24199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53800000000000003,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.30199999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.42099999999999999,"15min":0.42199999999999999},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69599999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.32400000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.9610000000000001,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.60699999999999998}]},{"osd":4,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63200000000000001,"15min":0.63900000000000001},"min":{"1min":0.45600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.85399999999999998,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86199999999999999,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.58699999999999997}]},{"osd":5,"last update":"Fri Apr 5 03:03:29 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.875},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64100000000000001,"15min":0.64300000000000002},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93999999999999995,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":0.70599999999999996}]},{"osd":6,"last update":"Fri Apr 5 03:02:50 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.628,"15min":0.621},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.629,"15min":0.63600000000000001},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76500000000000001,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.84099999999999997}]},{"osd":7,"last update":"Fri Apr 5 03:03:13 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59999999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.659,"5min":1.659,"15min":1.659},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.74099999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1314816,"data_stored":901172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 03:02:56 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35399999999999998,"15min":0.32400000000000001},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":0.53700000000000003,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.374,"15min":0.34000000000000002},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.54200000000000004,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.58999999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.46300000000000002,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.54000000000000004,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.55400000000000005}]},{"osd":3,"last update":"Fri Apr 5 03:03:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.441,"15min":0.44400000000000001},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.61299999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.50600000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.50600000000000001}]},{"osd":4,"last update":"Fri Apr 5 03:03:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.875,"15min":0.875},"last":0.80300000000000005}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.41699999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79500000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83599999999999997,"5min":1.018,"15min":1.018},"last":0.73199999999999998}]},{"osd":6,"last update":"Fri Apr 5 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.42499999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81499999999999995,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.64600000000000002,"15min":0.64500000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.76400000000000001}]},{"osd":7,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63500000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74099999999999999,"5min":0.875,"15min":0.875},"last":0.749},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.69699999999999995,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80600000000000005,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.82899999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411386,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":310816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51000000000000001,"5min":0.623,"15min":0.623},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39000000000000001,"15min":0.39400000000000002},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.53000000000000003,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.26000000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.39500000000000002,"15min":0.40500000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57599999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.54200000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39400000000000002}]},{"osd":3,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.44500000000000001,"15min":0.45900000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.55800000000000005,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.43099999999999999,"15min":0.45100000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.52900000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.29999999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:03:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.41299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.83699999999999997,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86799999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80800000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.66600000000000004,"15min":0.60899999999999999},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":12.968999999999999,"5min":12.968999999999999,"15min":12.968999999999999},"last":0.65000000000000002}]},{"osd":6,"last update":"Fri Apr 5 03:02:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.83399999999999996,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.45400000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73599999999999999,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.58699999999999997}]},{"osd":7,"last update":"Fri Apr 5 03:03:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.628}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":310302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39100000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.433,"15min":0.443},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81499999999999995,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.32900000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.45200000000000001,"15min":0.44900000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.85999999999999999,"5min":1.54,"15min":1.54},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44900000000000001,"15min":0.45700000000000002},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":1.468,"15min":1.468},"last":0.35499999999999998}]},{"osd":2,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.48299999999999998,"15min":0.496},"min":{"1min":0.28399999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.94899999999999995,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.38500000000000001}]},{"osd":4,"last update":"Fri Apr 5 03:03:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.434,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.96899999999999997,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.79100000000000004}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.432,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":1.002,"15min":1.002},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63600000000000001,"15min":0.64300000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.82199999999999995}]},{"osd":6,"last update":"Fri Apr 5 03:02:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.63900000000000001,"15min":0.64700000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.873,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89400000000000002,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.68500000000000005}]},{"osd":7,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.63700000000000001,"15min":0.64100000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46800000000000003,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.75800000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986225,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":737280,"data_stored":311910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.56100000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.442},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54700000000000004,"15min":0.56000000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66700000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56100000000000005}]},{"osd":1,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.753,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55200000000000005,"15min":0.56200000000000006},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.67700000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.59399999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.64600000000000002,"15min":0.57299999999999995},"min":{"1min":0.372,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":11.239000000000001,"5min":11.239000000000001,"15min":11.239000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57199999999999995,"15min":0.58999999999999997},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69499999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.49399999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.67900000000000005,"15min":0.61499999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.624},{"interface":"front","average":{"1min":1.133,"5min":0.69099999999999995,"15min":0.625},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.221,"5min":11.221,"15min":11.221},"last":0.51500000000000001}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.40200000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.432},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.623,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.54100000000000004}]},{"osd":6,"last update":"Fri Apr 5 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.45500000000000002,"15min":0.44700000000000001},"min":{"1min":0.34399999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45700000000000002,"15min":0.44900000000000001},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66900000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.45800000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:03:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.49299999999999999,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.58599999999999997,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.35899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.56999999999999995,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.38100000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724588,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":720896,"data_stored":310852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74399999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.41799999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.39600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.78400000000000003,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.63700000000000001,"15min":0.58099999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.2070000000000007,"5min":9.2070000000000007,"15min":9.2070000000000007},"last":0.49299999999999999}]},{"osd":2,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.443,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.55100000000000005,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.81599999999999995,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.57399999999999995}]},{"osd":3,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.75,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.56699999999999995,"15min":0.55700000000000005},"min":{"1min":0.437,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78100000000000003,"5min":1.329,"15min":1.329},"last":0.53000000000000003}]},{"osd":4,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.40899999999999997,"15min":0.38100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.63},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.42999999999999999,"15min":0.40200000000000002},"min":{"1min":0.32300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.71499999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.51900000000000002}]},{"osd":6,"last update":"Fri Apr 5 03:02:49 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.46800000000000003,"15min":0.45900000000000002},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.47999999999999998,"15min":0.46600000000000003},"min":{"1min":0.38,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.65200000000000002}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81999999999999995,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77000000000000002,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.60399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:03:53.610 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph status --format=json 2024-04-05T03:03:53.875 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:54.307 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:54 smithi033 ceph-mon[29244]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:54.307 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:54 smithi033 ceph-mon[40008]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:54.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:54 smithi151 ceph-mon[32789]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:03:55.206 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:55 smithi033 ceph-mon[40008]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:55.206 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:55 smithi033 ceph-mon[29244]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:55.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:55 smithi151 ceph-mon[32789]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:55.730 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:03:55.730 INFO:teuthology.orchestra.run.smithi033.stdout:{"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":14,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":364,"monmap":{"epoch":3,"min_mon_release_name":"quincy","num_mons":3},"osdmap":{"epoch":70,"num_osds":8,"num_up_osds":8,"osd_up_since":1712286071,"num_in_osds":8,"osd_in_since":1712286054,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":592706,"bytes_used":2443526144,"bytes_avail":765516759040,"bytes_total":767960285184},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi033.rmzwzs","status":"up:active","gid":14718}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.33:8443/","prometheus":"http://172.21.15.33:9283/"}},"servicemap":{"epoch":12,"modified":"2024-04-05T03:02:46.044166+0000","services":{}},"progress_events":{}} 2024-04-05T03:03:56.352 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:56 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/945801807' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-05T03:03:56.352 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:56 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/945801807' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-05T03:03:56.380 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-04-05T03:03:56.380 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:03:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:56 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/945801807' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-05T03:03:56.644 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:03:57.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:57 smithi151 ceph-mon[32789]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:57.283 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:03:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:03:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:03:57.442 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:57 smithi033 ceph-mon[40008]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:57.442 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:57 smithi033 ceph-mon[29244]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:58.414 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:03:58.415 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:03:59.167 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":78,"stamp":"2024-04-05T03:03:58.059425+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386256,"kb_used_data":7396,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574960,"statfs":{"total":767960285184,"available":765516759040,"internally_reserved":0,"allocated":7573504,"data_stored":4259305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002512"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125656+0000","last_change":"2024-04-05T03:02:48.902639+0000","last_active":"2024-04-05T03:03:44.125656+0000","last_peered":"2024-04-05T03:03:44.125656+0000","last_clean":"2024-04-05T03:03:44.125656+0000","last_became_active":"2024-04-05T03:02:48.902301+0000","last_became_peered":"2024-04-05T03:02:48.902301+0000","last_unstale":"2024-04-05T03:03:44.125656+0000","last_undegraded":"2024-04-05T03:03:44.125656+0000","last_fullsized":"2024-04-05T03:03:44.125656+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:20:53.245725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122247+0000","last_change":"2024-04-05T03:02:48.457889+0000","last_active":"2024-04-05T03:03:44.122247+0000","last_peered":"2024-04-05T03:03:44.122247+0000","last_clean":"2024-04-05T03:03:44.122247+0000","last_became_active":"2024-04-05T03:02:48.457391+0000","last_became_peered":"2024-04-05T03:02:48.457391+0000","last_unstale":"2024-04-05T03:03:44.122247+0000","last_undegraded":"2024-04-05T03:03:44.122247+0000","last_fullsized":"2024-04-05T03:03:44.122247+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:38:13.647069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122541+0000","last_change":"2024-04-05T03:02:49.746985+0000","last_active":"2024-04-05T03:03:44.122541+0000","last_peered":"2024-04-05T03:03:44.122541+0000","last_clean":"2024-04-05T03:03:44.122541+0000","last_became_active":"2024-04-05T03:02:49.746610+0000","last_became_peered":"2024-04-05T03:02:49.746610+0000","last_unstale":"2024-04-05T03:03:44.122541+0000","last_undegraded":"2024-04-05T03:03:44.122541+0000","last_fullsized":"2024-04-05T03:03:44.122541+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:56:20.326917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122901+0000","last_change":"2024-04-05T03:02:48.459650+0000","last_active":"2024-04-05T03:03:44.122901+0000","last_peered":"2024-04-05T03:03:44.122901+0000","last_clean":"2024-04-05T03:03:44.122901+0000","last_became_active":"2024-04-05T03:02:48.459355+0000","last_became_peered":"2024-04-05T03:02:48.459355+0000","last_unstale":"2024-04-05T03:03:44.122901+0000","last_undegraded":"2024-04-05T03:03:44.122901+0000","last_fullsized":"2024-04-05T03:03:44.122901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:23:16.774935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125932+0000","last_change":"2024-04-05T03:02:48.644045+0000","last_active":"2024-04-05T03:03:44.125932+0000","last_peered":"2024-04-05T03:03:44.125932+0000","last_clean":"2024-04-05T03:03:44.125932+0000","last_became_active":"2024-04-05T03:02:48.643795+0000","last_became_peered":"2024-04-05T03:02:48.643795+0000","last_unstale":"2024-04-05T03:03:44.125932+0000","last_undegraded":"2024-04-05T03:03:44.125932+0000","last_fullsized":"2024-04-05T03:03:44.125932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:49:15.865276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122130+0000","last_change":"2024-04-05T03:02:48.458396+0000","last_active":"2024-04-05T03:03:44.122130+0000","last_peered":"2024-04-05T03:03:44.122130+0000","last_clean":"2024-04-05T03:03:44.122130+0000","last_became_active":"2024-04-05T03:02:48.458092+0000","last_became_peered":"2024-04-05T03:02:48.458092+0000","last_unstale":"2024-04-05T03:03:44.122130+0000","last_undegraded":"2024-04-05T03:03:44.122130+0000","last_fullsized":"2024-04-05T03:03:44.122130+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:43.973088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126011+0000","last_change":"2024-04-05T03:02:48.472069+0000","last_active":"2024-04-05T03:03:44.126011+0000","last_peered":"2024-04-05T03:03:44.126011+0000","last_clean":"2024-04-05T03:03:44.126011+0000","last_became_active":"2024-04-05T03:02:48.471739+0000","last_became_peered":"2024-04-05T03:02:48.471739+0000","last_unstale":"2024-04-05T03:03:44.126011+0000","last_undegraded":"2024-04-05T03:03:44.126011+0000","last_fullsized":"2024-04-05T03:03:44.126011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:44:49.173752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121975+0000","last_change":"2024-04-05T03:02:48.459793+0000","last_active":"2024-04-05T03:03:44.121975+0000","last_peered":"2024-04-05T03:03:44.121975+0000","last_clean":"2024-04-05T03:03:44.121975+0000","last_became_active":"2024-04-05T03:02:48.459518+0000","last_became_peered":"2024-04-05T03:02:48.459518+0000","last_unstale":"2024-04-05T03:03:44.121975+0000","last_undegraded":"2024-04-05T03:03:44.121975+0000","last_fullsized":"2024-04-05T03:03:44.121975+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:03:23.429419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127208+0000","last_change":"2024-04-05T03:02:48.473124+0000","last_active":"2024-04-05T03:03:44.127208+0000","last_peered":"2024-04-05T03:03:44.127208+0000","last_clean":"2024-04-05T03:03:44.127208+0000","last_became_active":"2024-04-05T03:02:48.472816+0000","last_became_peered":"2024-04-05T03:02:48.472816+0000","last_unstale":"2024-04-05T03:03:44.127208+0000","last_undegraded":"2024-04-05T03:03:44.127208+0000","last_fullsized":"2024-04-05T03:03:44.127208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:09:10.287592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121649+0000","last_change":"2024-04-05T03:02:48.457652+0000","last_active":"2024-04-05T03:03:44.121649+0000","last_peered":"2024-04-05T03:03:44.121649+0000","last_clean":"2024-04-05T03:03:44.121649+0000","last_became_active":"2024-04-05T03:02:48.457340+0000","last_became_peered":"2024-04-05T03:02:48.457340+0000","last_unstale":"2024-04-05T03:03:44.121649+0000","last_undegraded":"2024-04-05T03:03:44.121649+0000","last_fullsized":"2024-04-05T03:03:44.121649+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:04:12.013393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125917+0000","last_change":"2024-04-05T03:02:48.902933+0000","last_active":"2024-04-05T03:03:44.125917+0000","last_peered":"2024-04-05T03:03:44.125917+0000","last_clean":"2024-04-05T03:03:44.125917+0000","last_became_active":"2024-04-05T03:02:48.902583+0000","last_became_peered":"2024-04-05T03:02:48.902583+0000","last_unstale":"2024-04-05T03:03:44.125917+0000","last_undegraded":"2024-04-05T03:03:44.125917+0000","last_fullsized":"2024-04-05T03:03:44.125917+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:11:03.431547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121971+0000","last_change":"2024-04-05T03:02:48.465598+0000","last_active":"2024-04-05T03:03:44.121971+0000","last_peered":"2024-04-05T03:03:44.121971+0000","last_clean":"2024-04-05T03:03:44.121971+0000","last_became_active":"2024-04-05T03:02:48.465280+0000","last_became_peered":"2024-04-05T03:02:48.465280+0000","last_unstale":"2024-04-05T03:03:44.121971+0000","last_undegraded":"2024-04-05T03:03:44.121971+0000","last_fullsized":"2024-04-05T03:03:44.121971+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:27:18.537900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125053+0000","last_change":"2024-04-05T03:01:43.659234+0000","last_active":"2024-04-05T03:03:44.125053+0000","last_peered":"2024-04-05T03:03:44.125053+0000","last_clean":"2024-04-05T03:03:44.125053+0000","last_became_active":"2024-04-05T03:01:43.658801+0000","last_became_peered":"2024-04-05T03:01:43.658801+0000","last_unstale":"2024-04-05T03:03:44.125053+0000","last_undegraded":"2024-04-05T03:03:44.125053+0000","last_fullsized":"2024-04-05T03:03:44.125053+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:39:37.401263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122318+0000","last_change":"2024-04-05T03:02:50.003751+0000","last_active":"2024-04-05T03:03:44.122318+0000","last_peered":"2024-04-05T03:03:44.122318+0000","last_clean":"2024-04-05T03:03:44.122318+0000","last_became_active":"2024-04-05T03:02:50.003472+0000","last_became_peered":"2024-04-05T03:02:50.003472+0000","last_unstale":"2024-04-05T03:03:44.122318+0000","last_undegraded":"2024-04-05T03:03:44.122318+0000","last_fullsized":"2024-04-05T03:03:44.122318+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:46:41.938159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125410+0000","last_change":"2024-04-05T03:01:43.656498+0000","last_active":"2024-04-05T03:03:44.125410+0000","last_peered":"2024-04-05T03:03:44.125410+0000","last_clean":"2024-04-05T03:03:44.125410+0000","last_became_active":"2024-04-05T03:01:43.656159+0000","last_became_peered":"2024-04-05T03:01:43.656159+0000","last_unstale":"2024-04-05T03:03:44.125410+0000","last_undegraded":"2024-04-05T03:03:44.125410+0000","last_fullsized":"2024-04-05T03:03:44.125410+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:39.619074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123066+0000","last_change":"2024-04-05T03:02:48.458703+0000","last_active":"2024-04-05T03:03:44.123066+0000","last_peered":"2024-04-05T03:03:44.123066+0000","last_clean":"2024-04-05T03:03:44.123066+0000","last_became_active":"2024-04-05T03:02:48.457630+0000","last_became_peered":"2024-04-05T03:02:48.457630+0000","last_unstale":"2024-04-05T03:03:44.123066+0000","last_undegraded":"2024-04-05T03:03:44.123066+0000","last_fullsized":"2024-04-05T03:03:44.123066+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:42:21.323311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122148+0000","last_change":"2024-04-05T03:02:50.002775+0000","last_active":"2024-04-05T03:03:44.122148+0000","last_peered":"2024-04-05T03:03:44.122148+0000","last_clean":"2024-04-05T03:03:44.122148+0000","last_became_active":"2024-04-05T03:02:50.002356+0000","last_became_peered":"2024-04-05T03:02:50.002356+0000","last_unstale":"2024-04-05T03:03:44.122148+0000","last_undegraded":"2024-04-05T03:03:44.122148+0000","last_fullsized":"2024-04-05T03:03:44.122148+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:47:10.699733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125761+0000","last_change":"2024-04-05T03:02:48.456621+0000","last_active":"2024-04-05T03:03:44.125761+0000","last_peered":"2024-04-05T03:03:44.125761+0000","last_clean":"2024-04-05T03:03:44.125761+0000","last_became_active":"2024-04-05T03:02:48.455491+0000","last_became_peered":"2024-04-05T03:02:48.455491+0000","last_unstale":"2024-04-05T03:03:44.125761+0000","last_undegraded":"2024-04-05T03:03:44.125761+0000","last_fullsized":"2024-04-05T03:03:44.125761+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:38.283998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127688+0000","last_change":"2024-04-05T03:01:43.910231+0000","last_active":"2024-04-05T03:03:44.127688+0000","last_peered":"2024-04-05T03:03:44.127688+0000","last_clean":"2024-04-05T03:03:44.127688+0000","last_became_active":"2024-04-05T03:01:43.909886+0000","last_became_peered":"2024-04-05T03:01:43.909886+0000","last_unstale":"2024-04-05T03:03:44.127688+0000","last_undegraded":"2024-04-05T03:03:44.127688+0000","last_fullsized":"2024-04-05T03:03:44.127688+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:22:51.464507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122040+0000","last_change":"2024-04-05T03:02:49.746307+0000","last_active":"2024-04-05T03:03:44.122040+0000","last_peered":"2024-04-05T03:03:44.122040+0000","last_clean":"2024-04-05T03:03:44.122040+0000","last_became_active":"2024-04-05T03:02:49.745976+0000","last_became_peered":"2024-04-05T03:02:49.745976+0000","last_unstale":"2024-04-05T03:03:44.122040+0000","last_undegraded":"2024-04-05T03:03:44.122040+0000","last_fullsized":"2024-04-05T03:03:44.122040+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:49:15.142317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121543+0000","last_change":"2024-04-05T03:02:48.461046+0000","last_active":"2024-04-05T03:03:44.121543+0000","last_peered":"2024-04-05T03:03:44.121543+0000","last_clean":"2024-04-05T03:03:44.121543+0000","last_became_active":"2024-04-05T03:02:48.460678+0000","last_became_peered":"2024-04-05T03:02:48.460678+0000","last_unstale":"2024-04-05T03:03:44.121543+0000","last_undegraded":"2024-04-05T03:03:44.121543+0000","last_fullsized":"2024-04-05T03:03:44.121543+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:23.864420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126652+0000","last_change":"2024-04-05T03:01:43.910268+0000","last_active":"2024-04-05T03:03:44.126652+0000","last_peered":"2024-04-05T03:03:44.126652+0000","last_clean":"2024-04-05T03:03:44.126652+0000","last_became_active":"2024-04-05T03:01:43.909991+0000","last_became_peered":"2024-04-05T03:01:43.909991+0000","last_unstale":"2024-04-05T03:03:44.126652+0000","last_undegraded":"2024-04-05T03:03:44.126652+0000","last_fullsized":"2024-04-05T03:03:44.126652+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:12.008125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124460+0000","last_change":"2024-04-05T03:02:50.003073+0000","last_active":"2024-04-05T03:03:44.124460+0000","last_peered":"2024-04-05T03:03:44.124460+0000","last_clean":"2024-04-05T03:03:44.124460+0000","last_became_active":"2024-04-05T03:02:50.002726+0000","last_became_peered":"2024-04-05T03:02:50.002726+0000","last_unstale":"2024-04-05T03:03:44.124460+0000","last_undegraded":"2024-04-05T03:03:44.124460+0000","last_fullsized":"2024-04-05T03:03:44.124460+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:50:50.549731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122646+0000","last_change":"2024-04-05T03:02:48.460577+0000","last_active":"2024-04-05T03:03:44.122646+0000","last_peered":"2024-04-05T03:03:44.122646+0000","last_clean":"2024-04-05T03:03:44.122646+0000","last_became_active":"2024-04-05T03:02:48.460216+0000","last_became_peered":"2024-04-05T03:02:48.460216+0000","last_unstale":"2024-04-05T03:03:44.122646+0000","last_undegraded":"2024-04-05T03:03:44.122646+0000","last_fullsized":"2024-04-05T03:03:44.122646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:35:58.345475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126636+0000","last_change":"2024-04-05T03:01:14.837104+0000","last_active":"2024-04-05T03:03:44.126636+0000","last_peered":"2024-04-05T03:03:44.126636+0000","last_clean":"2024-04-05T03:03:44.126636+0000","last_became_active":"2024-04-05T03:01:14.836690+0000","last_became_peered":"2024-04-05T03:01:14.836690+0000","last_unstale":"2024-04-05T03:03:44.126636+0000","last_undegraded":"2024-04-05T03:03:44.126636+0000","last_fullsized":"2024-04-05T03:03:44.126636+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:48:28.544381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121869+0000","last_change":"2024-04-05T03:01:43.659091+0000","last_active":"2024-04-05T03:03:44.121869+0000","last_peered":"2024-04-05T03:03:44.121869+0000","last_clean":"2024-04-05T03:03:44.121869+0000","last_became_active":"2024-04-05T03:01:43.658783+0000","last_became_peered":"2024-04-05T03:01:43.658783+0000","last_unstale":"2024-04-05T03:03:44.121869+0000","last_undegraded":"2024-04-05T03:03:44.121869+0000","last_fullsized":"2024-04-05T03:03:44.121869+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:34:20.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125743+0000","last_change":"2024-04-05T03:02:49.773210+0000","last_active":"2024-04-05T03:03:44.125743+0000","last_peered":"2024-04-05T03:03:44.125743+0000","last_clean":"2024-04-05T03:03:44.125743+0000","last_became_active":"2024-04-05T03:02:49.772812+0000","last_became_peered":"2024-04-05T03:02:49.772812+0000","last_unstale":"2024-04-05T03:03:44.125743+0000","last_undegraded":"2024-04-05T03:03:44.125743+0000","last_fullsized":"2024-04-05T03:03:44.125743+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:29:41.425482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125536+0000","last_change":"2024-04-05T03:02:48.457064+0000","last_active":"2024-04-05T03:03:44.125536+0000","last_peered":"2024-04-05T03:03:44.125536+0000","last_clean":"2024-04-05T03:03:44.125536+0000","last_became_active":"2024-04-05T03:02:48.456787+0000","last_became_peered":"2024-04-05T03:02:48.456787+0000","last_unstale":"2024-04-05T03:03:44.125536+0000","last_undegraded":"2024-04-05T03:03:44.125536+0000","last_fullsized":"2024-04-05T03:03:44.125536+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:01:04.753701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126939+0000","last_change":"2024-04-05T03:01:43.910452+0000","last_active":"2024-04-05T03:03:44.126939+0000","last_peered":"2024-04-05T03:03:44.126939+0000","last_clean":"2024-04-05T03:03:44.126939+0000","last_became_active":"2024-04-05T03:01:43.909876+0000","last_became_peered":"2024-04-05T03:01:43.909876+0000","last_unstale":"2024-04-05T03:03:44.126939+0000","last_undegraded":"2024-04-05T03:03:44.126939+0000","last_fullsized":"2024-04-05T03:03:44.126939+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:07:36.880582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125680+0000","last_change":"2024-04-05T03:02:49.746957+0000","last_active":"2024-04-05T03:03:44.125680+0000","last_peered":"2024-04-05T03:03:44.125680+0000","last_clean":"2024-04-05T03:03:44.125680+0000","last_became_active":"2024-04-05T03:02:49.746064+0000","last_became_peered":"2024-04-05T03:02:49.746064+0000","last_unstale":"2024-04-05T03:03:44.125680+0000","last_undegraded":"2024-04-05T03:03:44.125680+0000","last_fullsized":"2024-04-05T03:03:44.125680+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:42:42.677796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126016+0000","last_change":"2024-04-05T03:02:48.455210+0000","last_active":"2024-04-05T03:03:44.126016+0000","last_peered":"2024-04-05T03:03:44.126016+0000","last_clean":"2024-04-05T03:03:44.126016+0000","last_became_active":"2024-04-05T03:02:48.453370+0000","last_became_peered":"2024-04-05T03:02:48.453370+0000","last_unstale":"2024-04-05T03:03:44.126016+0000","last_undegraded":"2024-04-05T03:03:44.126016+0000","last_fullsized":"2024-04-05T03:03:44.126016+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:50.364653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121829+0000","last_change":"2024-04-05T03:01:43.656346+0000","last_active":"2024-04-05T03:03:44.121829+0000","last_peered":"2024-04-05T03:03:44.121829+0000","last_clean":"2024-04-05T03:03:44.121829+0000","last_became_active":"2024-04-05T03:01:43.656045+0000","last_became_peered":"2024-04-05T03:01:43.656045+0000","last_unstale":"2024-04-05T03:03:44.121829+0000","last_undegraded":"2024-04-05T03:03:44.121829+0000","last_fullsized":"2024-04-05T03:03:44.121829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:40:55.612040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125054+0000","last_change":"2024-04-05T03:02:49.772316+0000","last_active":"2024-04-05T03:03:44.125054+0000","last_peered":"2024-04-05T03:03:44.125054+0000","last_clean":"2024-04-05T03:03:44.125054+0000","last_became_active":"2024-04-05T03:02:49.772060+0000","last_became_peered":"2024-04-05T03:02:49.772060+0000","last_unstale":"2024-04-05T03:03:44.125054+0000","last_undegraded":"2024-04-05T03:03:44.125054+0000","last_fullsized":"2024-04-05T03:03:44.125054+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:05:14.265331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122117+0000","last_change":"2024-04-05T03:02:48.457200+0000","last_active":"2024-04-05T03:03:44.122117+0000","last_peered":"2024-04-05T03:03:44.122117+0000","last_clean":"2024-04-05T03:03:44.122117+0000","last_became_active":"2024-04-05T03:02:48.456346+0000","last_became_peered":"2024-04-05T03:02:48.456346+0000","last_unstale":"2024-04-05T03:03:44.122117+0000","last_undegraded":"2024-04-05T03:03:44.122117+0000","last_fullsized":"2024-04-05T03:03:44.122117+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:22.908070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125596+0000","last_change":"2024-04-05T03:01:43.910340+0000","last_active":"2024-04-05T03:03:44.125596+0000","last_peered":"2024-04-05T03:03:44.125596+0000","last_clean":"2024-04-05T03:03:44.125596+0000","last_became_active":"2024-04-05T03:01:43.909987+0000","last_became_peered":"2024-04-05T03:01:43.909987+0000","last_unstale":"2024-04-05T03:03:44.125596+0000","last_undegraded":"2024-04-05T03:03:44.125596+0000","last_fullsized":"2024-04-05T03:03:44.125596+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:42.918552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125820+0000","last_change":"2024-04-05T03:02:49.469399+0000","last_active":"2024-04-05T03:03:44.125820+0000","last_peered":"2024-04-05T03:03:44.125820+0000","last_clean":"2024-04-05T03:03:44.125820+0000","last_became_active":"2024-04-05T03:02:49.469112+0000","last_became_peered":"2024-04-05T03:02:49.469112+0000","last_unstale":"2024-04-05T03:03:44.125820+0000","last_undegraded":"2024-04-05T03:03:44.125820+0000","last_fullsized":"2024-04-05T03:03:44.125820+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:59:24.415770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122553+0000","last_change":"2024-04-05T03:02:48.460772+0000","last_active":"2024-04-05T03:03:44.122553+0000","last_peered":"2024-04-05T03:03:44.122553+0000","last_clean":"2024-04-05T03:03:44.122553+0000","last_became_active":"2024-04-05T03:02:48.458509+0000","last_became_peered":"2024-04-05T03:02:48.458509+0000","last_unstale":"2024-04-05T03:03:44.122553+0000","last_undegraded":"2024-04-05T03:03:44.122553+0000","last_fullsized":"2024-04-05T03:03:44.122553+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:56:41.129273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126900+0000","last_change":"2024-04-05T03:02:48.902626+0000","last_active":"2024-04-05T03:03:44.126900+0000","last_peered":"2024-04-05T03:03:44.126900+0000","last_clean":"2024-04-05T03:03:44.126900+0000","last_became_active":"2024-04-05T03:02:48.902271+0000","last_became_peered":"2024-04-05T03:02:48.902271+0000","last_unstale":"2024-04-05T03:03:44.126900+0000","last_undegraded":"2024-04-05T03:03:44.126900+0000","last_fullsized":"2024-04-05T03:03:44.126900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:31:05.812000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125944+0000","last_change":"2024-04-05T03:02:48.448680+0000","last_active":"2024-04-05T03:03:44.125944+0000","last_peered":"2024-04-05T03:03:44.125944+0000","last_clean":"2024-04-05T03:03:44.125944+0000","last_became_active":"2024-04-05T03:02:48.448318+0000","last_became_peered":"2024-04-05T03:02:48.448318+0000","last_unstale":"2024-04-05T03:03:44.125944+0000","last_undegraded":"2024-04-05T03:03:44.125944+0000","last_fullsized":"2024-04-05T03:03:44.125944+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:17.475717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126861+0000","last_change":"2024-04-05T03:02:48.472192+0000","last_active":"2024-04-05T03:03:44.126861+0000","last_peered":"2024-04-05T03:03:44.126861+0000","last_clean":"2024-04-05T03:03:44.126861+0000","last_became_active":"2024-04-05T03:02:48.471904+0000","last_became_peered":"2024-04-05T03:02:48.471904+0000","last_unstale":"2024-04-05T03:03:44.126861+0000","last_undegraded":"2024-04-05T03:03:44.126861+0000","last_fullsized":"2024-04-05T03:03:44.126861+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:38:38.241850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124325+0000","last_change":"2024-04-05T03:02:48.459300+0000","last_active":"2024-04-05T03:03:44.124325+0000","last_peered":"2024-04-05T03:03:44.124325+0000","last_clean":"2024-04-05T03:03:44.124325+0000","last_became_active":"2024-04-05T03:02:48.458886+0000","last_became_peered":"2024-04-05T03:02:48.458886+0000","last_unstale":"2024-04-05T03:03:44.124325+0000","last_undegraded":"2024-04-05T03:03:44.124325+0000","last_fullsized":"2024-04-05T03:03:44.124325+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:55:07.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126488+0000","last_change":"2024-04-05T03:02:48.904058+0000","last_active":"2024-04-05T03:03:44.126488+0000","last_peered":"2024-04-05T03:03:44.126488+0000","last_clean":"2024-04-05T03:03:44.126488+0000","last_became_active":"2024-04-05T03:02:48.903756+0000","last_became_peered":"2024-04-05T03:02:48.903756+0000","last_unstale":"2024-04-05T03:03:44.126488+0000","last_undegraded":"2024-04-05T03:03:44.126488+0000","last_fullsized":"2024-04-05T03:03:44.126488+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:11:54.123968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123183+0000","last_change":"2024-04-05T03:02:48.457251+0000","last_active":"2024-04-05T03:03:44.123183+0000","last_peered":"2024-04-05T03:03:44.123183+0000","last_clean":"2024-04-05T03:03:44.123183+0000","last_became_active":"2024-04-05T03:02:48.456950+0000","last_became_peered":"2024-04-05T03:02:48.456950+0000","last_unstale":"2024-04-05T03:03:44.123183+0000","last_undegraded":"2024-04-05T03:03:44.123183+0000","last_fullsized":"2024-04-05T03:03:44.123183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:56:54.797317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121818+0000","last_change":"2024-04-05T03:02:49.473606+0000","last_active":"2024-04-05T03:03:44.121818+0000","last_peered":"2024-04-05T03:03:44.121818+0000","last_clean":"2024-04-05T03:03:44.121818+0000","last_became_active":"2024-04-05T03:02:49.473288+0000","last_became_peered":"2024-04-05T03:02:49.473288+0000","last_unstale":"2024-04-05T03:03:44.121818+0000","last_undegraded":"2024-04-05T03:03:44.121818+0000","last_fullsized":"2024-04-05T03:03:44.121818+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:16:58.265293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121870+0000","last_change":"2024-04-05T03:02:48.461876+0000","last_active":"2024-04-05T03:03:44.121870+0000","last_peered":"2024-04-05T03:03:44.121870+0000","last_clean":"2024-04-05T03:03:44.121870+0000","last_became_active":"2024-04-05T03:02:48.461576+0000","last_became_peered":"2024-04-05T03:02:48.461576+0000","last_unstale":"2024-04-05T03:03:44.121870+0000","last_undegraded":"2024-04-05T03:03:44.121870+0000","last_fullsized":"2024-04-05T03:03:44.121870+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:24:53.545330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.159681+0000","last_change":"2024-04-05T03:03:45.159681+0000","last_active":"2024-04-05T03:03:45.159681+0000","last_peered":"2024-04-05T03:03:45.159681+0000","last_clean":"2024-04-05T03:03:45.159681+0000","last_became_active":"2024-04-05T03:03:45.159381+0000","last_became_peered":"2024-04-05T03:03:45.159381+0000","last_unstale":"2024-04-05T03:03:45.159681+0000","last_undegraded":"2024-04-05T03:03:45.159681+0000","last_fullsized":"2024-04-05T03:03:45.159681+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:47:51.454737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127784+0000","last_change":"2024-04-05T03:02:48.461252+0000","last_active":"2024-04-05T03:03:44.127784+0000","last_peered":"2024-04-05T03:03:44.127784+0000","last_clean":"2024-04-05T03:03:44.127784+0000","last_became_active":"2024-04-05T03:02:48.460982+0000","last_became_peered":"2024-04-05T03:02:48.460982+0000","last_unstale":"2024-04-05T03:03:44.127784+0000","last_undegraded":"2024-04-05T03:03:44.127784+0000","last_fullsized":"2024-04-05T03:03:44.127784+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:03.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125800+0000","last_change":"2024-04-05T03:02:48.472006+0000","last_active":"2024-04-05T03:03:44.125800+0000","last_peered":"2024-04-05T03:03:44.125800+0000","last_clean":"2024-04-05T03:03:44.125800+0000","last_became_active":"2024-04-05T03:02:48.471696+0000","last_became_peered":"2024-04-05T03:02:48.471696+0000","last_unstale":"2024-04-05T03:03:44.125800+0000","last_undegraded":"2024-04-05T03:03:44.125800+0000","last_fullsized":"2024-04-05T03:03:44.125800+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:31:37.135819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127543+0000","last_change":"2024-04-05T03:02:48.457001+0000","last_active":"2024-04-05T03:03:44.127543+0000","last_peered":"2024-04-05T03:03:44.127543+0000","last_clean":"2024-04-05T03:03:44.127543+0000","last_became_active":"2024-04-05T03:02:48.456334+0000","last_became_peered":"2024-04-05T03:02:48.456334+0000","last_unstale":"2024-04-05T03:03:44.127543+0000","last_undegraded":"2024-04-05T03:03:44.127543+0000","last_fullsized":"2024-04-05T03:03:44.127543+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:16:57.078617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121741+0000","last_change":"2024-04-05T03:02:48.903134+0000","last_active":"2024-04-05T03:03:44.121741+0000","last_peered":"2024-04-05T03:03:44.121741+0000","last_clean":"2024-04-05T03:03:44.121741+0000","last_became_active":"2024-04-05T03:02:48.902836+0000","last_became_peered":"2024-04-05T03:02:48.902836+0000","last_unstale":"2024-04-05T03:03:44.121741+0000","last_undegraded":"2024-04-05T03:03:44.121741+0000","last_fullsized":"2024-04-05T03:03:44.121741+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:08:54.250101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127935+0000","last_change":"2024-04-05T03:02:48.457358+0000","last_active":"2024-04-05T03:03:44.127935+0000","last_peered":"2024-04-05T03:03:44.127935+0000","last_clean":"2024-04-05T03:03:44.127935+0000","last_became_active":"2024-04-05T03:02:48.456987+0000","last_became_peered":"2024-04-05T03:02:48.456987+0000","last_unstale":"2024-04-05T03:03:44.127935+0000","last_undegraded":"2024-04-05T03:03:44.127935+0000","last_fullsized":"2024-04-05T03:03:44.127935+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:22.057798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123071+0000","last_change":"2024-04-05T03:02:50.003001+0000","last_active":"2024-04-05T03:03:44.123071+0000","last_peered":"2024-04-05T03:03:44.123071+0000","last_clean":"2024-04-05T03:03:44.123071+0000","last_became_active":"2024-04-05T03:02:50.002734+0000","last_became_peered":"2024-04-05T03:02:50.002734+0000","last_unstale":"2024-04-05T03:03:44.123071+0000","last_undegraded":"2024-04-05T03:03:44.123071+0000","last_fullsized":"2024-04-05T03:03:44.123071+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:52:55.958187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123875+0000","last_change":"2024-04-05T03:02:48.457497+0000","last_active":"2024-04-05T03:03:44.123875+0000","last_peered":"2024-04-05T03:03:44.123875+0000","last_clean":"2024-04-05T03:03:44.123875+0000","last_became_active":"2024-04-05T03:02:48.457161+0000","last_became_peered":"2024-04-05T03:02:48.457161+0000","last_unstale":"2024-04-05T03:03:44.123875+0000","last_undegraded":"2024-04-05T03:03:44.123875+0000","last_fullsized":"2024-04-05T03:03:44.123875+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:31:49.521313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121819+0000","last_change":"2024-04-05T03:02:50.003556+0000","last_active":"2024-04-05T03:03:44.121819+0000","last_peered":"2024-04-05T03:03:44.121819+0000","last_clean":"2024-04-05T03:03:44.121819+0000","last_became_active":"2024-04-05T03:02:50.003271+0000","last_became_peered":"2024-04-05T03:02:50.003271+0000","last_unstale":"2024-04-05T03:03:44.121819+0000","last_undegraded":"2024-04-05T03:03:44.121819+0000","last_fullsized":"2024-04-05T03:03:44.121819+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:00:59.126639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126848+0000","last_change":"2024-04-05T03:02:48.460853+0000","last_active":"2024-04-05T03:03:44.126848+0000","last_peered":"2024-04-05T03:03:44.126848+0000","last_clean":"2024-04-05T03:03:44.126848+0000","last_became_active":"2024-04-05T03:02:48.460557+0000","last_became_peered":"2024-04-05T03:02:48.460557+0000","last_unstale":"2024-04-05T03:03:44.126848+0000","last_undegraded":"2024-04-05T03:03:44.126848+0000","last_fullsized":"2024-04-05T03:03:44.126848+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:57.261993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122011+0000","last_change":"2024-04-05T03:02:48.902752+0000","last_active":"2024-04-05T03:03:44.122011+0000","last_peered":"2024-04-05T03:03:44.122011+0000","last_clean":"2024-04-05T03:03:44.122011+0000","last_became_active":"2024-04-05T03:02:48.902396+0000","last_became_peered":"2024-04-05T03:02:48.902396+0000","last_unstale":"2024-04-05T03:03:44.122011+0000","last_undegraded":"2024-04-05T03:03:44.122011+0000","last_fullsized":"2024-04-05T03:03:44.122011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:45:00.227236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121540+0000","last_change":"2024-04-05T03:02:48.461320+0000","last_active":"2024-04-05T03:03:44.121540+0000","last_peered":"2024-04-05T03:03:44.121540+0000","last_clean":"2024-04-05T03:03:44.121540+0000","last_became_active":"2024-04-05T03:02:48.461034+0000","last_became_peered":"2024-04-05T03:02:48.461034+0000","last_unstale":"2024-04-05T03:03:44.121540+0000","last_undegraded":"2024-04-05T03:03:44.121540+0000","last_fullsized":"2024-04-05T03:03:44.121540+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:34:08.199664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122756+0000","last_change":"2024-04-05T03:02:50.002729+0000","last_active":"2024-04-05T03:03:44.122756+0000","last_peered":"2024-04-05T03:03:44.122756+0000","last_clean":"2024-04-05T03:03:44.122756+0000","last_became_active":"2024-04-05T03:02:50.002342+0000","last_became_peered":"2024-04-05T03:02:50.002342+0000","last_unstale":"2024-04-05T03:03:44.122756+0000","last_undegraded":"2024-04-05T03:03:44.122756+0000","last_fullsized":"2024-04-05T03:03:44.122756+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:51:56.223511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127209+0000","last_change":"2024-04-05T03:02:48.459896+0000","last_active":"2024-04-05T03:03:44.127209+0000","last_peered":"2024-04-05T03:03:44.127209+0000","last_clean":"2024-04-05T03:03:44.127209+0000","last_became_active":"2024-04-05T03:02:48.458108+0000","last_became_peered":"2024-04-05T03:02:48.458108+0000","last_unstale":"2024-04-05T03:03:44.127209+0000","last_undegraded":"2024-04-05T03:03:44.127209+0000","last_fullsized":"2024-04-05T03:03:44.127209+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:41:12.902400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126553+0000","last_change":"2024-04-05T03:02:48.903143+0000","last_active":"2024-04-05T03:03:44.126553+0000","last_peered":"2024-04-05T03:03:44.126553+0000","last_clean":"2024-04-05T03:03:44.126553+0000","last_became_active":"2024-04-05T03:02:48.902265+0000","last_became_peered":"2024-04-05T03:02:48.902265+0000","last_unstale":"2024-04-05T03:03:44.126553+0000","last_undegraded":"2024-04-05T03:03:44.126553+0000","last_fullsized":"2024-04-05T03:03:44.126553+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:05:56.179576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126604+0000","last_change":"2024-04-05T03:02:48.455833+0000","last_active":"2024-04-05T03:03:44.126604+0000","last_peered":"2024-04-05T03:03:44.126604+0000","last_clean":"2024-04-05T03:03:44.126604+0000","last_became_active":"2024-04-05T03:02:48.455185+0000","last_became_peered":"2024-04-05T03:02:48.455185+0000","last_unstale":"2024-04-05T03:03:44.126604+0000","last_undegraded":"2024-04-05T03:03:44.126604+0000","last_fullsized":"2024-04-05T03:03:44.126604+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:02.621579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124971+0000","last_change":"2024-04-05T03:02:49.746606+0000","last_active":"2024-04-05T03:03:44.124971+0000","last_peered":"2024-04-05T03:03:44.124971+0000","last_clean":"2024-04-05T03:03:44.124971+0000","last_became_active":"2024-04-05T03:02:49.746308+0000","last_became_peered":"2024-04-05T03:02:49.746308+0000","last_unstale":"2024-04-05T03:03:44.124971+0000","last_undegraded":"2024-04-05T03:03:44.124971+0000","last_fullsized":"2024-04-05T03:03:44.124971+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:07:39.965218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127576+0000","last_change":"2024-04-05T03:02:48.456697+0000","last_active":"2024-04-05T03:03:44.127576+0000","last_peered":"2024-04-05T03:03:44.127576+0000","last_clean":"2024-04-05T03:03:44.127576+0000","last_became_active":"2024-04-05T03:02:48.456140+0000","last_became_peered":"2024-04-05T03:02:48.456140+0000","last_unstale":"2024-04-05T03:03:44.127576+0000","last_undegraded":"2024-04-05T03:03:44.127576+0000","last_fullsized":"2024-04-05T03:03:44.127576+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:57:31.290707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122585+0000","last_change":"2024-04-05T03:02:50.002658+0000","last_active":"2024-04-05T03:03:44.122585+0000","last_peered":"2024-04-05T03:03:44.122585+0000","last_clean":"2024-04-05T03:03:44.122585+0000","last_became_active":"2024-04-05T03:02:50.002383+0000","last_became_peered":"2024-04-05T03:02:50.002383+0000","last_unstale":"2024-04-05T03:03:44.122585+0000","last_undegraded":"2024-04-05T03:03:44.122585+0000","last_fullsized":"2024-04-05T03:03:44.122585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:05:34.117134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121968+0000","last_change":"2024-04-05T03:02:48.457005+0000","last_active":"2024-04-05T03:03:44.121968+0000","last_peered":"2024-04-05T03:03:44.121968+0000","last_clean":"2024-04-05T03:03:44.121968+0000","last_became_active":"2024-04-05T03:02:48.456670+0000","last_became_peered":"2024-04-05T03:02:48.456670+0000","last_unstale":"2024-04-05T03:03:44.121968+0000","last_undegraded":"2024-04-05T03:03:44.121968+0000","last_fullsized":"2024-04-05T03:03:44.121968+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:45:12.114782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122420+0000","last_change":"2024-04-05T03:02:49.471218+0000","last_active":"2024-04-05T03:03:44.122420+0000","last_peered":"2024-04-05T03:03:44.122420+0000","last_clean":"2024-04-05T03:03:44.122420+0000","last_became_active":"2024-04-05T03:02:49.470918+0000","last_became_peered":"2024-04-05T03:02:49.470918+0000","last_unstale":"2024-04-05T03:03:44.122420+0000","last_undegraded":"2024-04-05T03:03:44.122420+0000","last_fullsized":"2024-04-05T03:03:44.122420+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:48:19.807934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122364+0000","last_change":"2024-04-05T03:02:48.463560+0000","last_active":"2024-04-05T03:03:44.122364+0000","last_peered":"2024-04-05T03:03:44.122364+0000","last_clean":"2024-04-05T03:03:44.122364+0000","last_became_active":"2024-04-05T03:02:48.463245+0000","last_became_peered":"2024-04-05T03:02:48.463245+0000","last_unstale":"2024-04-05T03:03:44.122364+0000","last_undegraded":"2024-04-05T03:03:44.122364+0000","last_fullsized":"2024-04-05T03:03:44.122364+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:47:16.914625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.134193+0000","last_change":"2024-04-05T03:03:45.134193+0000","last_active":"2024-04-05T03:03:45.134193+0000","last_peered":"2024-04-05T03:03:45.134193+0000","last_clean":"2024-04-05T03:03:45.134193+0000","last_became_active":"2024-04-05T03:03:45.133891+0000","last_became_peered":"2024-04-05T03:03:45.133891+0000","last_unstale":"2024-04-05T03:03:45.134193+0000","last_undegraded":"2024-04-05T03:03:45.134193+0000","last_fullsized":"2024-04-05T03:03:45.134193+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:56:06.527860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122675+0000","last_change":"2024-04-05T03:02:48.457628+0000","last_active":"2024-04-05T03:03:44.122675+0000","last_peered":"2024-04-05T03:03:44.122675+0000","last_clean":"2024-04-05T03:03:44.122675+0000","last_became_active":"2024-04-05T03:02:48.457318+0000","last_became_peered":"2024-04-05T03:02:48.457318+0000","last_unstale":"2024-04-05T03:03:44.122675+0000","last_undegraded":"2024-04-05T03:03:44.122675+0000","last_fullsized":"2024-04-05T03:03:44.122675+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:32.231625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121893+0000","last_change":"2024-04-05T03:02:50.004105+0000","last_active":"2024-04-05T03:03:44.121893+0000","last_peered":"2024-04-05T03:03:44.121893+0000","last_clean":"2024-04-05T03:03:44.121893+0000","last_became_active":"2024-04-05T03:02:50.003693+0000","last_became_peered":"2024-04-05T03:02:50.003693+0000","last_unstale":"2024-04-05T03:03:44.121893+0000","last_undegraded":"2024-04-05T03:03:44.121893+0000","last_fullsized":"2024-04-05T03:03:44.121893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:02.044564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125825+0000","last_change":"2024-04-05T03:02:48.464554+0000","last_active":"2024-04-05T03:03:44.125825+0000","last_peered":"2024-04-05T03:03:44.125825+0000","last_clean":"2024-04-05T03:03:44.125825+0000","last_became_active":"2024-04-05T03:02:48.463923+0000","last_became_peered":"2024-04-05T03:02:48.463923+0000","last_unstale":"2024-04-05T03:03:44.125825+0000","last_undegraded":"2024-04-05T03:03:44.125825+0000","last_fullsized":"2024-04-05T03:03:44.125825+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:56.699626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.256204+0000","last_change":"2024-04-05T03:03:45.256204+0000","last_active":"2024-04-05T03:03:45.256204+0000","last_peered":"2024-04-05T03:03:45.256204+0000","last_clean":"2024-04-05T03:03:45.256204+0000","last_became_active":"2024-04-05T03:03:45.255812+0000","last_became_peered":"2024-04-05T03:03:45.255812+0000","last_unstale":"2024-04-05T03:03:45.256204+0000","last_undegraded":"2024-04-05T03:03:45.256204+0000","last_fullsized":"2024-04-05T03:03:45.256204+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:15:31.145449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125800+0000","last_change":"2024-04-05T03:02:48.456415+0000","last_active":"2024-04-05T03:03:44.125800+0000","last_peered":"2024-04-05T03:03:44.125800+0000","last_clean":"2024-04-05T03:03:44.125800+0000","last_became_active":"2024-04-05T03:02:48.455959+0000","last_became_peered":"2024-04-05T03:02:48.455959+0000","last_unstale":"2024-04-05T03:03:44.125800+0000","last_undegraded":"2024-04-05T03:03:44.125800+0000","last_fullsized":"2024-04-05T03:03:44.125800+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:52:33.266219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298660,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446492,"statfs":{"total":95995035648,"available":95689207808,"internally_reserved":0,"allocated":1335296,"data_stored":902348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.38400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51000000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60999999999999999}]},{"osd":1,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78700000000000003,"5min":0.878,"15min":0.878},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.379,"5min":0.371,"15min":0.371},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.55700000000000005}]},{"osd":2,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.43099999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55400000000000005,"15min":0.55300000000000005},"min":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.5},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72999999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.53100000000000003}]},{"osd":4,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48199999999999998,"15min":0.48399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70099999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.57199999999999995}]},{"osd":5,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.502,"15min":0.50600000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.77400000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77100000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.63400000000000001}]},{"osd":6,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.39800000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.432,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78800000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.64800000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":901958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.71099999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.70899999999999996,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.438}]},{"osd":1,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.39200000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.42699999999999999}]},{"osd":2,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.46200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69399999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.504},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.64900000000000002,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.48499999999999999}]},{"osd":3,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57299999999999995,"15min":0.56000000000000005},"min":{"1min":0.438,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.55300000000000005}]},{"osd":4,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.44,"15min":0.42999999999999999},"min":{"1min":0.315,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.45100000000000001,"15min":0.42699999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.64500000000000002}]},{"osd":5,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.46400000000000002,"15min":0.44700000000000001},"min":{"1min":0.36499999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.626,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.52600000000000002,"15min":0.499},"min":{"1min":0.40400000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.60299999999999998}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.624}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":708608,"data_stored":309947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.316},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.56599999999999995,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.39400000000000002,"15min":0.34999999999999998},"min":{"1min":0.27100000000000002,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.59699999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.40600000000000003}]},{"osd":2,"last update":"Fri Apr 5 03:03:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40600000000000003,"15min":0.39400000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.621,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.39200000000000002,"15min":0.37},"min":{"1min":0.24199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53800000000000003,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.66600000000000004}]},{"osd":3,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41099999999999998,"15min":0.41899999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.60599999999999998,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46800000000000003,"15min":0.46400000000000002},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60699999999999998,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.55400000000000005}]},{"osd":4,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63200000000000001,"15min":0.63900000000000001},"min":{"1min":0.45600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.85399999999999998,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86199999999999999,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.64400000000000002}]},{"osd":5,"last update":"Fri Apr 5 03:03:29 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64100000000000001,"15min":0.64300000000000002},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93999999999999995,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":0.78100000000000003}]},{"osd":6,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.63400000000000001,"15min":0.623},"min":{"1min":0.375,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0029999999999999,"5min":1.74,"15min":1.74},"last":0.873},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.629,"15min":0.63600000000000001},"min":{"1min":0.45400000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.59399999999999997}]},{"osd":7,"last update":"Fri Apr 5 03:03:13 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59999999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.659,"5min":1.659,"15min":1.659},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.52100000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1314816,"data_stored":901172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 03:02:56 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.35399999999999998,"15min":0.32400000000000001},"min":{"1min":0.22700000000000001,"5min":0.186,"15min":0.186},"max":{"1min":0.53700000000000003,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.374,"15min":0.34000000000000002},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.54200000000000004,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.58299999999999996}]},{"osd":2,"last update":"Fri Apr 5 03:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.46300000000000002,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.371,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.54000000000000004,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.48299999999999998}]},{"osd":3,"last update":"Fri Apr 5 03:03:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.441,"15min":0.44400000000000001},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.61299999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.50600000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54200000000000004}]},{"osd":4,"last update":"Fri Apr 5 03:03:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.875,"15min":0.875},"last":0.72999999999999998}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.41699999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79500000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83599999999999997,"5min":1.018,"15min":1.018},"last":0.81699999999999995}]},{"osd":6,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60599999999999998,"15min":0.60799999999999998},"min":{"1min":0.438,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81399999999999995,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.82399999999999995,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.76100000000000001}]},{"osd":7,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63500000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74099999999999999,"5min":0.875,"15min":0.875},"last":0.749},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.69699999999999995,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80600000000000005,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.78200000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":310816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51000000000000001,"5min":0.623,"15min":0.623},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39000000000000001,"15min":0.39400000000000002},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.53000000000000003,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.26600000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.39500000000000002,"15min":0.40500000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57599999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.54200000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.307}]},{"osd":3,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.44500000000000001,"15min":0.45900000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.55800000000000005,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.43099999999999999,"15min":0.45100000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.52900000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.375}]},{"osd":4,"last update":"Fri Apr 5 03:03:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.41299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.83699999999999997,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86799999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.65500000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80800000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.66600000000000004,"15min":0.60899999999999999},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":12.968999999999999,"5min":12.968999999999999,"15min":12.968999999999999},"last":0.67500000000000004}]},{"osd":6,"last update":"Fri Apr 5 03:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.59999999999999998,"15min":0.59699999999999998},"min":{"1min":0.45900000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.46899999999999997,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.628}]},{"osd":7,"last update":"Fri Apr 5 03:03:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.626,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.60399999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":310302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39100000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.433,"15min":0.443},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81499999999999995,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.54200000000000004}]},{"osd":1,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.45200000000000001,"15min":0.44900000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.85999999999999999,"5min":1.54,"15min":1.54},"last":0.434},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44900000000000001,"15min":0.45700000000000002},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":1.468,"15min":1.468},"last":0.46999999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.48299999999999998,"15min":0.496},"min":{"1min":0.28399999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.94899999999999995,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.52400000000000002}]},{"osd":4,"last update":"Fri Apr 5 03:03:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.434,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.96899999999999997,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.86899999999999999}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.432,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":1.002,"15min":1.002},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63600000000000001,"15min":0.64300000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.80400000000000005}]},{"osd":6,"last update":"Fri Apr 5 03:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.64900000000000002,"15min":0.65600000000000003},"min":{"1min":0.45500000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89300000000000002,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.71499999999999997}]},{"osd":7,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.63700000000000001,"15min":0.64100000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46800000000000003,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.72599999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":737280,"data_stored":311910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.56100000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54700000000000004,"15min":0.56000000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66700000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56799999999999995}]},{"osd":1,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.753,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55200000000000005,"15min":0.56200000000000006},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.67700000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.44}]},{"osd":2,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.64600000000000002,"15min":0.57299999999999995},"min":{"1min":0.372,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":11.239000000000001,"5min":11.239000000000001,"15min":11.239000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57199999999999995,"15min":0.58999999999999997},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69499999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.40400000000000003}]},{"osd":3,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.67900000000000005,"15min":0.61499999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.133,"5min":0.69099999999999995,"15min":0.625},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.221,"5min":11.221,"15min":11.221},"last":0.51700000000000002}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.40200000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.432},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.623,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.48099999999999998}]},{"osd":6,"last update":"Fri Apr 5 03:02:54 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.45500000000000002,"15min":0.44700000000000001},"min":{"1min":0.34399999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45700000000000002,"15min":0.44900000000000001},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66900000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.53300000000000003}]},{"osd":7,"last update":"Fri Apr 5 03:03:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.49299999999999999,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.58599999999999997,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.497},{"interface":"front","average":{"1min":0.438,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.35899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.56999999999999995,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.55300000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724589,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":720896,"data_stored":310852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74399999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.747}]},{"osd":1,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.39600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.78400000000000003,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.63700000000000001,"15min":0.58099999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.2070000000000007,"5min":9.2070000000000007,"15min":9.2070000000000007},"last":0.68700000000000006}]},{"osd":2,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.443,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.55100000000000005,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.81599999999999995,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.72399999999999998}]},{"osd":3,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.75,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.56699999999999995,"15min":0.55700000000000005},"min":{"1min":0.437,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78100000000000003,"5min":1.329,"15min":1.329},"last":0.67600000000000005}]},{"osd":4,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.40899999999999997,"15min":0.38100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.42999999999999999,"15min":0.40200000000000002},"min":{"1min":0.32300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.71499999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.58799999999999997}]},{"osd":6,"last update":"Fri Apr 5 03:03:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48699999999999999,"15min":0.46500000000000002},"min":{"1min":0.38,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.5,"15min":0.47199999999999998},"min":{"1min":0.38900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.81499999999999995,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.625}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81999999999999995,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77000000000000002,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.77100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:03:59.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:59 smithi033 ceph-mon[40008]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:59.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:03:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:59 smithi033 ceph-mon[29244]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:59.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:03:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:59.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:59 smithi151 ceph-mon[32789]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:03:59.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:03:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:00.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:00 smithi033 ceph-mon[29244]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:04:00.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:00 smithi033 ceph-mon[40008]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:04:00.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:00 smithi151 ceph-mon[32789]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:04:01.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:01 smithi033 ceph-mon[29244]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:01.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:01 smithi033 ceph-mon[40008]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:01 smithi151 ceph-mon[32789]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:02.172 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:04:02.445 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:02.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:02 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:02] "GET /metrics HTTP/1.1" 200 239239 "" "Prometheus/2.43.0" 2024-04-05T03:04:03.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:03 smithi033 ceph-mon[29244]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:03.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:03 smithi033 ceph-mon[40008]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:03.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:03 smithi151 ceph-mon[32789]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:04.308 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:04:04.310 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:04:04.962 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":81,"stamp":"2024-04-05T03:04:04.061390+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386256,"kb_used_data":7396,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574960,"statfs":{"total":767960285184,"available":765516759040,"internally_reserved":0,"allocated":7573504,"data_stored":4259305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003227"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125656+0000","last_change":"2024-04-05T03:02:48.902639+0000","last_active":"2024-04-05T03:03:44.125656+0000","last_peered":"2024-04-05T03:03:44.125656+0000","last_clean":"2024-04-05T03:03:44.125656+0000","last_became_active":"2024-04-05T03:02:48.902301+0000","last_became_peered":"2024-04-05T03:02:48.902301+0000","last_unstale":"2024-04-05T03:03:44.125656+0000","last_undegraded":"2024-04-05T03:03:44.125656+0000","last_fullsized":"2024-04-05T03:03:44.125656+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:20:53.245725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122247+0000","last_change":"2024-04-05T03:02:48.457889+0000","last_active":"2024-04-05T03:03:44.122247+0000","last_peered":"2024-04-05T03:03:44.122247+0000","last_clean":"2024-04-05T03:03:44.122247+0000","last_became_active":"2024-04-05T03:02:48.457391+0000","last_became_peered":"2024-04-05T03:02:48.457391+0000","last_unstale":"2024-04-05T03:03:44.122247+0000","last_undegraded":"2024-04-05T03:03:44.122247+0000","last_fullsized":"2024-04-05T03:03:44.122247+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:38:13.647069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122541+0000","last_change":"2024-04-05T03:02:49.746985+0000","last_active":"2024-04-05T03:03:44.122541+0000","last_peered":"2024-04-05T03:03:44.122541+0000","last_clean":"2024-04-05T03:03:44.122541+0000","last_became_active":"2024-04-05T03:02:49.746610+0000","last_became_peered":"2024-04-05T03:02:49.746610+0000","last_unstale":"2024-04-05T03:03:44.122541+0000","last_undegraded":"2024-04-05T03:03:44.122541+0000","last_fullsized":"2024-04-05T03:03:44.122541+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:56:20.326917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122901+0000","last_change":"2024-04-05T03:02:48.459650+0000","last_active":"2024-04-05T03:03:44.122901+0000","last_peered":"2024-04-05T03:03:44.122901+0000","last_clean":"2024-04-05T03:03:44.122901+0000","last_became_active":"2024-04-05T03:02:48.459355+0000","last_became_peered":"2024-04-05T03:02:48.459355+0000","last_unstale":"2024-04-05T03:03:44.122901+0000","last_undegraded":"2024-04-05T03:03:44.122901+0000","last_fullsized":"2024-04-05T03:03:44.122901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:23:16.774935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125932+0000","last_change":"2024-04-05T03:02:48.644045+0000","last_active":"2024-04-05T03:03:44.125932+0000","last_peered":"2024-04-05T03:03:44.125932+0000","last_clean":"2024-04-05T03:03:44.125932+0000","last_became_active":"2024-04-05T03:02:48.643795+0000","last_became_peered":"2024-04-05T03:02:48.643795+0000","last_unstale":"2024-04-05T03:03:44.125932+0000","last_undegraded":"2024-04-05T03:03:44.125932+0000","last_fullsized":"2024-04-05T03:03:44.125932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:49:15.865276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122130+0000","last_change":"2024-04-05T03:02:48.458396+0000","last_active":"2024-04-05T03:03:44.122130+0000","last_peered":"2024-04-05T03:03:44.122130+0000","last_clean":"2024-04-05T03:03:44.122130+0000","last_became_active":"2024-04-05T03:02:48.458092+0000","last_became_peered":"2024-04-05T03:02:48.458092+0000","last_unstale":"2024-04-05T03:03:44.122130+0000","last_undegraded":"2024-04-05T03:03:44.122130+0000","last_fullsized":"2024-04-05T03:03:44.122130+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:43.973088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126011+0000","last_change":"2024-04-05T03:02:48.472069+0000","last_active":"2024-04-05T03:03:44.126011+0000","last_peered":"2024-04-05T03:03:44.126011+0000","last_clean":"2024-04-05T03:03:44.126011+0000","last_became_active":"2024-04-05T03:02:48.471739+0000","last_became_peered":"2024-04-05T03:02:48.471739+0000","last_unstale":"2024-04-05T03:03:44.126011+0000","last_undegraded":"2024-04-05T03:03:44.126011+0000","last_fullsized":"2024-04-05T03:03:44.126011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:44:49.173752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121975+0000","last_change":"2024-04-05T03:02:48.459793+0000","last_active":"2024-04-05T03:03:44.121975+0000","last_peered":"2024-04-05T03:03:44.121975+0000","last_clean":"2024-04-05T03:03:44.121975+0000","last_became_active":"2024-04-05T03:02:48.459518+0000","last_became_peered":"2024-04-05T03:02:48.459518+0000","last_unstale":"2024-04-05T03:03:44.121975+0000","last_undegraded":"2024-04-05T03:03:44.121975+0000","last_fullsized":"2024-04-05T03:03:44.121975+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:03:23.429419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127208+0000","last_change":"2024-04-05T03:02:48.473124+0000","last_active":"2024-04-05T03:03:44.127208+0000","last_peered":"2024-04-05T03:03:44.127208+0000","last_clean":"2024-04-05T03:03:44.127208+0000","last_became_active":"2024-04-05T03:02:48.472816+0000","last_became_peered":"2024-04-05T03:02:48.472816+0000","last_unstale":"2024-04-05T03:03:44.127208+0000","last_undegraded":"2024-04-05T03:03:44.127208+0000","last_fullsized":"2024-04-05T03:03:44.127208+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:09:10.287592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121649+0000","last_change":"2024-04-05T03:02:48.457652+0000","last_active":"2024-04-05T03:03:44.121649+0000","last_peered":"2024-04-05T03:03:44.121649+0000","last_clean":"2024-04-05T03:03:44.121649+0000","last_became_active":"2024-04-05T03:02:48.457340+0000","last_became_peered":"2024-04-05T03:02:48.457340+0000","last_unstale":"2024-04-05T03:03:44.121649+0000","last_undegraded":"2024-04-05T03:03:44.121649+0000","last_fullsized":"2024-04-05T03:03:44.121649+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:04:12.013393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125917+0000","last_change":"2024-04-05T03:02:48.902933+0000","last_active":"2024-04-05T03:03:44.125917+0000","last_peered":"2024-04-05T03:03:44.125917+0000","last_clean":"2024-04-05T03:03:44.125917+0000","last_became_active":"2024-04-05T03:02:48.902583+0000","last_became_peered":"2024-04-05T03:02:48.902583+0000","last_unstale":"2024-04-05T03:03:44.125917+0000","last_undegraded":"2024-04-05T03:03:44.125917+0000","last_fullsized":"2024-04-05T03:03:44.125917+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:11:03.431547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121971+0000","last_change":"2024-04-05T03:02:48.465598+0000","last_active":"2024-04-05T03:03:44.121971+0000","last_peered":"2024-04-05T03:03:44.121971+0000","last_clean":"2024-04-05T03:03:44.121971+0000","last_became_active":"2024-04-05T03:02:48.465280+0000","last_became_peered":"2024-04-05T03:02:48.465280+0000","last_unstale":"2024-04-05T03:03:44.121971+0000","last_undegraded":"2024-04-05T03:03:44.121971+0000","last_fullsized":"2024-04-05T03:03:44.121971+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:27:18.537900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125053+0000","last_change":"2024-04-05T03:01:43.659234+0000","last_active":"2024-04-05T03:03:44.125053+0000","last_peered":"2024-04-05T03:03:44.125053+0000","last_clean":"2024-04-05T03:03:44.125053+0000","last_became_active":"2024-04-05T03:01:43.658801+0000","last_became_peered":"2024-04-05T03:01:43.658801+0000","last_unstale":"2024-04-05T03:03:44.125053+0000","last_undegraded":"2024-04-05T03:03:44.125053+0000","last_fullsized":"2024-04-05T03:03:44.125053+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:39:37.401263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122318+0000","last_change":"2024-04-05T03:02:50.003751+0000","last_active":"2024-04-05T03:03:44.122318+0000","last_peered":"2024-04-05T03:03:44.122318+0000","last_clean":"2024-04-05T03:03:44.122318+0000","last_became_active":"2024-04-05T03:02:50.003472+0000","last_became_peered":"2024-04-05T03:02:50.003472+0000","last_unstale":"2024-04-05T03:03:44.122318+0000","last_undegraded":"2024-04-05T03:03:44.122318+0000","last_fullsized":"2024-04-05T03:03:44.122318+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:46:41.938159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125410+0000","last_change":"2024-04-05T03:01:43.656498+0000","last_active":"2024-04-05T03:03:44.125410+0000","last_peered":"2024-04-05T03:03:44.125410+0000","last_clean":"2024-04-05T03:03:44.125410+0000","last_became_active":"2024-04-05T03:01:43.656159+0000","last_became_peered":"2024-04-05T03:01:43.656159+0000","last_unstale":"2024-04-05T03:03:44.125410+0000","last_undegraded":"2024-04-05T03:03:44.125410+0000","last_fullsized":"2024-04-05T03:03:44.125410+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:39.619074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123066+0000","last_change":"2024-04-05T03:02:48.458703+0000","last_active":"2024-04-05T03:03:44.123066+0000","last_peered":"2024-04-05T03:03:44.123066+0000","last_clean":"2024-04-05T03:03:44.123066+0000","last_became_active":"2024-04-05T03:02:48.457630+0000","last_became_peered":"2024-04-05T03:02:48.457630+0000","last_unstale":"2024-04-05T03:03:44.123066+0000","last_undegraded":"2024-04-05T03:03:44.123066+0000","last_fullsized":"2024-04-05T03:03:44.123066+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:42:21.323311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122148+0000","last_change":"2024-04-05T03:02:50.002775+0000","last_active":"2024-04-05T03:03:44.122148+0000","last_peered":"2024-04-05T03:03:44.122148+0000","last_clean":"2024-04-05T03:03:44.122148+0000","last_became_active":"2024-04-05T03:02:50.002356+0000","last_became_peered":"2024-04-05T03:02:50.002356+0000","last_unstale":"2024-04-05T03:03:44.122148+0000","last_undegraded":"2024-04-05T03:03:44.122148+0000","last_fullsized":"2024-04-05T03:03:44.122148+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:47:10.699733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125761+0000","last_change":"2024-04-05T03:02:48.456621+0000","last_active":"2024-04-05T03:03:44.125761+0000","last_peered":"2024-04-05T03:03:44.125761+0000","last_clean":"2024-04-05T03:03:44.125761+0000","last_became_active":"2024-04-05T03:02:48.455491+0000","last_became_peered":"2024-04-05T03:02:48.455491+0000","last_unstale":"2024-04-05T03:03:44.125761+0000","last_undegraded":"2024-04-05T03:03:44.125761+0000","last_fullsized":"2024-04-05T03:03:44.125761+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:38.283998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127688+0000","last_change":"2024-04-05T03:01:43.910231+0000","last_active":"2024-04-05T03:03:44.127688+0000","last_peered":"2024-04-05T03:03:44.127688+0000","last_clean":"2024-04-05T03:03:44.127688+0000","last_became_active":"2024-04-05T03:01:43.909886+0000","last_became_peered":"2024-04-05T03:01:43.909886+0000","last_unstale":"2024-04-05T03:03:44.127688+0000","last_undegraded":"2024-04-05T03:03:44.127688+0000","last_fullsized":"2024-04-05T03:03:44.127688+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:22:51.464507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122040+0000","last_change":"2024-04-05T03:02:49.746307+0000","last_active":"2024-04-05T03:03:44.122040+0000","last_peered":"2024-04-05T03:03:44.122040+0000","last_clean":"2024-04-05T03:03:44.122040+0000","last_became_active":"2024-04-05T03:02:49.745976+0000","last_became_peered":"2024-04-05T03:02:49.745976+0000","last_unstale":"2024-04-05T03:03:44.122040+0000","last_undegraded":"2024-04-05T03:03:44.122040+0000","last_fullsized":"2024-04-05T03:03:44.122040+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:49:15.142317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121543+0000","last_change":"2024-04-05T03:02:48.461046+0000","last_active":"2024-04-05T03:03:44.121543+0000","last_peered":"2024-04-05T03:03:44.121543+0000","last_clean":"2024-04-05T03:03:44.121543+0000","last_became_active":"2024-04-05T03:02:48.460678+0000","last_became_peered":"2024-04-05T03:02:48.460678+0000","last_unstale":"2024-04-05T03:03:44.121543+0000","last_undegraded":"2024-04-05T03:03:44.121543+0000","last_fullsized":"2024-04-05T03:03:44.121543+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:23.864420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126652+0000","last_change":"2024-04-05T03:01:43.910268+0000","last_active":"2024-04-05T03:03:44.126652+0000","last_peered":"2024-04-05T03:03:44.126652+0000","last_clean":"2024-04-05T03:03:44.126652+0000","last_became_active":"2024-04-05T03:01:43.909991+0000","last_became_peered":"2024-04-05T03:01:43.909991+0000","last_unstale":"2024-04-05T03:03:44.126652+0000","last_undegraded":"2024-04-05T03:03:44.126652+0000","last_fullsized":"2024-04-05T03:03:44.126652+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:12.008125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124460+0000","last_change":"2024-04-05T03:02:50.003073+0000","last_active":"2024-04-05T03:03:44.124460+0000","last_peered":"2024-04-05T03:03:44.124460+0000","last_clean":"2024-04-05T03:03:44.124460+0000","last_became_active":"2024-04-05T03:02:50.002726+0000","last_became_peered":"2024-04-05T03:02:50.002726+0000","last_unstale":"2024-04-05T03:03:44.124460+0000","last_undegraded":"2024-04-05T03:03:44.124460+0000","last_fullsized":"2024-04-05T03:03:44.124460+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:50:50.549731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122646+0000","last_change":"2024-04-05T03:02:48.460577+0000","last_active":"2024-04-05T03:03:44.122646+0000","last_peered":"2024-04-05T03:03:44.122646+0000","last_clean":"2024-04-05T03:03:44.122646+0000","last_became_active":"2024-04-05T03:02:48.460216+0000","last_became_peered":"2024-04-05T03:02:48.460216+0000","last_unstale":"2024-04-05T03:03:44.122646+0000","last_undegraded":"2024-04-05T03:03:44.122646+0000","last_fullsized":"2024-04-05T03:03:44.122646+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:35:58.345475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126636+0000","last_change":"2024-04-05T03:01:14.837104+0000","last_active":"2024-04-05T03:03:44.126636+0000","last_peered":"2024-04-05T03:03:44.126636+0000","last_clean":"2024-04-05T03:03:44.126636+0000","last_became_active":"2024-04-05T03:01:14.836690+0000","last_became_peered":"2024-04-05T03:01:14.836690+0000","last_unstale":"2024-04-05T03:03:44.126636+0000","last_undegraded":"2024-04-05T03:03:44.126636+0000","last_fullsized":"2024-04-05T03:03:44.126636+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:48:28.544381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121869+0000","last_change":"2024-04-05T03:01:43.659091+0000","last_active":"2024-04-05T03:03:44.121869+0000","last_peered":"2024-04-05T03:03:44.121869+0000","last_clean":"2024-04-05T03:03:44.121869+0000","last_became_active":"2024-04-05T03:01:43.658783+0000","last_became_peered":"2024-04-05T03:01:43.658783+0000","last_unstale":"2024-04-05T03:03:44.121869+0000","last_undegraded":"2024-04-05T03:03:44.121869+0000","last_fullsized":"2024-04-05T03:03:44.121869+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:34:20.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125743+0000","last_change":"2024-04-05T03:02:49.773210+0000","last_active":"2024-04-05T03:03:44.125743+0000","last_peered":"2024-04-05T03:03:44.125743+0000","last_clean":"2024-04-05T03:03:44.125743+0000","last_became_active":"2024-04-05T03:02:49.772812+0000","last_became_peered":"2024-04-05T03:02:49.772812+0000","last_unstale":"2024-04-05T03:03:44.125743+0000","last_undegraded":"2024-04-05T03:03:44.125743+0000","last_fullsized":"2024-04-05T03:03:44.125743+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:29:41.425482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125536+0000","last_change":"2024-04-05T03:02:48.457064+0000","last_active":"2024-04-05T03:03:44.125536+0000","last_peered":"2024-04-05T03:03:44.125536+0000","last_clean":"2024-04-05T03:03:44.125536+0000","last_became_active":"2024-04-05T03:02:48.456787+0000","last_became_peered":"2024-04-05T03:02:48.456787+0000","last_unstale":"2024-04-05T03:03:44.125536+0000","last_undegraded":"2024-04-05T03:03:44.125536+0000","last_fullsized":"2024-04-05T03:03:44.125536+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:01:04.753701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126939+0000","last_change":"2024-04-05T03:01:43.910452+0000","last_active":"2024-04-05T03:03:44.126939+0000","last_peered":"2024-04-05T03:03:44.126939+0000","last_clean":"2024-04-05T03:03:44.126939+0000","last_became_active":"2024-04-05T03:01:43.909876+0000","last_became_peered":"2024-04-05T03:01:43.909876+0000","last_unstale":"2024-04-05T03:03:44.126939+0000","last_undegraded":"2024-04-05T03:03:44.126939+0000","last_fullsized":"2024-04-05T03:03:44.126939+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:07:36.880582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125680+0000","last_change":"2024-04-05T03:02:49.746957+0000","last_active":"2024-04-05T03:03:44.125680+0000","last_peered":"2024-04-05T03:03:44.125680+0000","last_clean":"2024-04-05T03:03:44.125680+0000","last_became_active":"2024-04-05T03:02:49.746064+0000","last_became_peered":"2024-04-05T03:02:49.746064+0000","last_unstale":"2024-04-05T03:03:44.125680+0000","last_undegraded":"2024-04-05T03:03:44.125680+0000","last_fullsized":"2024-04-05T03:03:44.125680+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:42:42.677796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126016+0000","last_change":"2024-04-05T03:02:48.455210+0000","last_active":"2024-04-05T03:03:44.126016+0000","last_peered":"2024-04-05T03:03:44.126016+0000","last_clean":"2024-04-05T03:03:44.126016+0000","last_became_active":"2024-04-05T03:02:48.453370+0000","last_became_peered":"2024-04-05T03:02:48.453370+0000","last_unstale":"2024-04-05T03:03:44.126016+0000","last_undegraded":"2024-04-05T03:03:44.126016+0000","last_fullsized":"2024-04-05T03:03:44.126016+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:50.364653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121829+0000","last_change":"2024-04-05T03:01:43.656346+0000","last_active":"2024-04-05T03:03:44.121829+0000","last_peered":"2024-04-05T03:03:44.121829+0000","last_clean":"2024-04-05T03:03:44.121829+0000","last_became_active":"2024-04-05T03:01:43.656045+0000","last_became_peered":"2024-04-05T03:01:43.656045+0000","last_unstale":"2024-04-05T03:03:44.121829+0000","last_undegraded":"2024-04-05T03:03:44.121829+0000","last_fullsized":"2024-04-05T03:03:44.121829+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:40:55.612040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125054+0000","last_change":"2024-04-05T03:02:49.772316+0000","last_active":"2024-04-05T03:03:44.125054+0000","last_peered":"2024-04-05T03:03:44.125054+0000","last_clean":"2024-04-05T03:03:44.125054+0000","last_became_active":"2024-04-05T03:02:49.772060+0000","last_became_peered":"2024-04-05T03:02:49.772060+0000","last_unstale":"2024-04-05T03:03:44.125054+0000","last_undegraded":"2024-04-05T03:03:44.125054+0000","last_fullsized":"2024-04-05T03:03:44.125054+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:05:14.265331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122117+0000","last_change":"2024-04-05T03:02:48.457200+0000","last_active":"2024-04-05T03:03:44.122117+0000","last_peered":"2024-04-05T03:03:44.122117+0000","last_clean":"2024-04-05T03:03:44.122117+0000","last_became_active":"2024-04-05T03:02:48.456346+0000","last_became_peered":"2024-04-05T03:02:48.456346+0000","last_unstale":"2024-04-05T03:03:44.122117+0000","last_undegraded":"2024-04-05T03:03:44.122117+0000","last_fullsized":"2024-04-05T03:03:44.122117+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:22.908070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125596+0000","last_change":"2024-04-05T03:01:43.910340+0000","last_active":"2024-04-05T03:03:44.125596+0000","last_peered":"2024-04-05T03:03:44.125596+0000","last_clean":"2024-04-05T03:03:44.125596+0000","last_became_active":"2024-04-05T03:01:43.909987+0000","last_became_peered":"2024-04-05T03:01:43.909987+0000","last_unstale":"2024-04-05T03:03:44.125596+0000","last_undegraded":"2024-04-05T03:03:44.125596+0000","last_fullsized":"2024-04-05T03:03:44.125596+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:42.918552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125820+0000","last_change":"2024-04-05T03:02:49.469399+0000","last_active":"2024-04-05T03:03:44.125820+0000","last_peered":"2024-04-05T03:03:44.125820+0000","last_clean":"2024-04-05T03:03:44.125820+0000","last_became_active":"2024-04-05T03:02:49.469112+0000","last_became_peered":"2024-04-05T03:02:49.469112+0000","last_unstale":"2024-04-05T03:03:44.125820+0000","last_undegraded":"2024-04-05T03:03:44.125820+0000","last_fullsized":"2024-04-05T03:03:44.125820+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:59:24.415770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122553+0000","last_change":"2024-04-05T03:02:48.460772+0000","last_active":"2024-04-05T03:03:44.122553+0000","last_peered":"2024-04-05T03:03:44.122553+0000","last_clean":"2024-04-05T03:03:44.122553+0000","last_became_active":"2024-04-05T03:02:48.458509+0000","last_became_peered":"2024-04-05T03:02:48.458509+0000","last_unstale":"2024-04-05T03:03:44.122553+0000","last_undegraded":"2024-04-05T03:03:44.122553+0000","last_fullsized":"2024-04-05T03:03:44.122553+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:56:41.129273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126900+0000","last_change":"2024-04-05T03:02:48.902626+0000","last_active":"2024-04-05T03:03:44.126900+0000","last_peered":"2024-04-05T03:03:44.126900+0000","last_clean":"2024-04-05T03:03:44.126900+0000","last_became_active":"2024-04-05T03:02:48.902271+0000","last_became_peered":"2024-04-05T03:02:48.902271+0000","last_unstale":"2024-04-05T03:03:44.126900+0000","last_undegraded":"2024-04-05T03:03:44.126900+0000","last_fullsized":"2024-04-05T03:03:44.126900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:31:05.812000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125944+0000","last_change":"2024-04-05T03:02:48.448680+0000","last_active":"2024-04-05T03:03:44.125944+0000","last_peered":"2024-04-05T03:03:44.125944+0000","last_clean":"2024-04-05T03:03:44.125944+0000","last_became_active":"2024-04-05T03:02:48.448318+0000","last_became_peered":"2024-04-05T03:02:48.448318+0000","last_unstale":"2024-04-05T03:03:44.125944+0000","last_undegraded":"2024-04-05T03:03:44.125944+0000","last_fullsized":"2024-04-05T03:03:44.125944+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:59:17.475717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126861+0000","last_change":"2024-04-05T03:02:48.472192+0000","last_active":"2024-04-05T03:03:44.126861+0000","last_peered":"2024-04-05T03:03:44.126861+0000","last_clean":"2024-04-05T03:03:44.126861+0000","last_became_active":"2024-04-05T03:02:48.471904+0000","last_became_peered":"2024-04-05T03:02:48.471904+0000","last_unstale":"2024-04-05T03:03:44.126861+0000","last_undegraded":"2024-04-05T03:03:44.126861+0000","last_fullsized":"2024-04-05T03:03:44.126861+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:38:38.241850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124325+0000","last_change":"2024-04-05T03:02:48.459300+0000","last_active":"2024-04-05T03:03:44.124325+0000","last_peered":"2024-04-05T03:03:44.124325+0000","last_clean":"2024-04-05T03:03:44.124325+0000","last_became_active":"2024-04-05T03:02:48.458886+0000","last_became_peered":"2024-04-05T03:02:48.458886+0000","last_unstale":"2024-04-05T03:03:44.124325+0000","last_undegraded":"2024-04-05T03:03:44.124325+0000","last_fullsized":"2024-04-05T03:03:44.124325+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:55:07.328688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126488+0000","last_change":"2024-04-05T03:02:48.904058+0000","last_active":"2024-04-05T03:03:44.126488+0000","last_peered":"2024-04-05T03:03:44.126488+0000","last_clean":"2024-04-05T03:03:44.126488+0000","last_became_active":"2024-04-05T03:02:48.903756+0000","last_became_peered":"2024-04-05T03:02:48.903756+0000","last_unstale":"2024-04-05T03:03:44.126488+0000","last_undegraded":"2024-04-05T03:03:44.126488+0000","last_fullsized":"2024-04-05T03:03:44.126488+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:11:54.123968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123183+0000","last_change":"2024-04-05T03:02:48.457251+0000","last_active":"2024-04-05T03:03:44.123183+0000","last_peered":"2024-04-05T03:03:44.123183+0000","last_clean":"2024-04-05T03:03:44.123183+0000","last_became_active":"2024-04-05T03:02:48.456950+0000","last_became_peered":"2024-04-05T03:02:48.456950+0000","last_unstale":"2024-04-05T03:03:44.123183+0000","last_undegraded":"2024-04-05T03:03:44.123183+0000","last_fullsized":"2024-04-05T03:03:44.123183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:56:54.797317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121818+0000","last_change":"2024-04-05T03:02:49.473606+0000","last_active":"2024-04-05T03:03:44.121818+0000","last_peered":"2024-04-05T03:03:44.121818+0000","last_clean":"2024-04-05T03:03:44.121818+0000","last_became_active":"2024-04-05T03:02:49.473288+0000","last_became_peered":"2024-04-05T03:02:49.473288+0000","last_unstale":"2024-04-05T03:03:44.121818+0000","last_undegraded":"2024-04-05T03:03:44.121818+0000","last_fullsized":"2024-04-05T03:03:44.121818+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:16:58.265293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121870+0000","last_change":"2024-04-05T03:02:48.461876+0000","last_active":"2024-04-05T03:03:44.121870+0000","last_peered":"2024-04-05T03:03:44.121870+0000","last_clean":"2024-04-05T03:03:44.121870+0000","last_became_active":"2024-04-05T03:02:48.461576+0000","last_became_peered":"2024-04-05T03:02:48.461576+0000","last_unstale":"2024-04-05T03:03:44.121870+0000","last_undegraded":"2024-04-05T03:03:44.121870+0000","last_fullsized":"2024-04-05T03:03:44.121870+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:24:53.545330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.159681+0000","last_change":"2024-04-05T03:03:45.159681+0000","last_active":"2024-04-05T03:03:45.159681+0000","last_peered":"2024-04-05T03:03:45.159681+0000","last_clean":"2024-04-05T03:03:45.159681+0000","last_became_active":"2024-04-05T03:03:45.159381+0000","last_became_peered":"2024-04-05T03:03:45.159381+0000","last_unstale":"2024-04-05T03:03:45.159681+0000","last_undegraded":"2024-04-05T03:03:45.159681+0000","last_fullsized":"2024-04-05T03:03:45.159681+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:47:51.454737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127784+0000","last_change":"2024-04-05T03:02:48.461252+0000","last_active":"2024-04-05T03:03:44.127784+0000","last_peered":"2024-04-05T03:03:44.127784+0000","last_clean":"2024-04-05T03:03:44.127784+0000","last_became_active":"2024-04-05T03:02:48.460982+0000","last_became_peered":"2024-04-05T03:02:48.460982+0000","last_unstale":"2024-04-05T03:03:44.127784+0000","last_undegraded":"2024-04-05T03:03:44.127784+0000","last_fullsized":"2024-04-05T03:03:44.127784+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:03.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125800+0000","last_change":"2024-04-05T03:02:48.472006+0000","last_active":"2024-04-05T03:03:44.125800+0000","last_peered":"2024-04-05T03:03:44.125800+0000","last_clean":"2024-04-05T03:03:44.125800+0000","last_became_active":"2024-04-05T03:02:48.471696+0000","last_became_peered":"2024-04-05T03:02:48.471696+0000","last_unstale":"2024-04-05T03:03:44.125800+0000","last_undegraded":"2024-04-05T03:03:44.125800+0000","last_fullsized":"2024-04-05T03:03:44.125800+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:31:37.135819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127543+0000","last_change":"2024-04-05T03:02:48.457001+0000","last_active":"2024-04-05T03:03:44.127543+0000","last_peered":"2024-04-05T03:03:44.127543+0000","last_clean":"2024-04-05T03:03:44.127543+0000","last_became_active":"2024-04-05T03:02:48.456334+0000","last_became_peered":"2024-04-05T03:02:48.456334+0000","last_unstale":"2024-04-05T03:03:44.127543+0000","last_undegraded":"2024-04-05T03:03:44.127543+0000","last_fullsized":"2024-04-05T03:03:44.127543+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:16:57.078617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121741+0000","last_change":"2024-04-05T03:02:48.903134+0000","last_active":"2024-04-05T03:03:44.121741+0000","last_peered":"2024-04-05T03:03:44.121741+0000","last_clean":"2024-04-05T03:03:44.121741+0000","last_became_active":"2024-04-05T03:02:48.902836+0000","last_became_peered":"2024-04-05T03:02:48.902836+0000","last_unstale":"2024-04-05T03:03:44.121741+0000","last_undegraded":"2024-04-05T03:03:44.121741+0000","last_fullsized":"2024-04-05T03:03:44.121741+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:08:54.250101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127935+0000","last_change":"2024-04-05T03:02:48.457358+0000","last_active":"2024-04-05T03:03:44.127935+0000","last_peered":"2024-04-05T03:03:44.127935+0000","last_clean":"2024-04-05T03:03:44.127935+0000","last_became_active":"2024-04-05T03:02:48.456987+0000","last_became_peered":"2024-04-05T03:02:48.456987+0000","last_unstale":"2024-04-05T03:03:44.127935+0000","last_undegraded":"2024-04-05T03:03:44.127935+0000","last_fullsized":"2024-04-05T03:03:44.127935+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:22.057798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123071+0000","last_change":"2024-04-05T03:02:50.003001+0000","last_active":"2024-04-05T03:03:44.123071+0000","last_peered":"2024-04-05T03:03:44.123071+0000","last_clean":"2024-04-05T03:03:44.123071+0000","last_became_active":"2024-04-05T03:02:50.002734+0000","last_became_peered":"2024-04-05T03:02:50.002734+0000","last_unstale":"2024-04-05T03:03:44.123071+0000","last_undegraded":"2024-04-05T03:03:44.123071+0000","last_fullsized":"2024-04-05T03:03:44.123071+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:52:55.958187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.123875+0000","last_change":"2024-04-05T03:02:48.457497+0000","last_active":"2024-04-05T03:03:44.123875+0000","last_peered":"2024-04-05T03:03:44.123875+0000","last_clean":"2024-04-05T03:03:44.123875+0000","last_became_active":"2024-04-05T03:02:48.457161+0000","last_became_peered":"2024-04-05T03:02:48.457161+0000","last_unstale":"2024-04-05T03:03:44.123875+0000","last_undegraded":"2024-04-05T03:03:44.123875+0000","last_fullsized":"2024-04-05T03:03:44.123875+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:31:49.521313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121819+0000","last_change":"2024-04-05T03:02:50.003556+0000","last_active":"2024-04-05T03:03:44.121819+0000","last_peered":"2024-04-05T03:03:44.121819+0000","last_clean":"2024-04-05T03:03:44.121819+0000","last_became_active":"2024-04-05T03:02:50.003271+0000","last_became_peered":"2024-04-05T03:02:50.003271+0000","last_unstale":"2024-04-05T03:03:44.121819+0000","last_undegraded":"2024-04-05T03:03:44.121819+0000","last_fullsized":"2024-04-05T03:03:44.121819+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:00:59.126639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126848+0000","last_change":"2024-04-05T03:02:48.460853+0000","last_active":"2024-04-05T03:03:44.126848+0000","last_peered":"2024-04-05T03:03:44.126848+0000","last_clean":"2024-04-05T03:03:44.126848+0000","last_became_active":"2024-04-05T03:02:48.460557+0000","last_became_peered":"2024-04-05T03:02:48.460557+0000","last_unstale":"2024-04-05T03:03:44.126848+0000","last_undegraded":"2024-04-05T03:03:44.126848+0000","last_fullsized":"2024-04-05T03:03:44.126848+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:57.261993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122011+0000","last_change":"2024-04-05T03:02:48.902752+0000","last_active":"2024-04-05T03:03:44.122011+0000","last_peered":"2024-04-05T03:03:44.122011+0000","last_clean":"2024-04-05T03:03:44.122011+0000","last_became_active":"2024-04-05T03:02:48.902396+0000","last_became_peered":"2024-04-05T03:02:48.902396+0000","last_unstale":"2024-04-05T03:03:44.122011+0000","last_undegraded":"2024-04-05T03:03:44.122011+0000","last_fullsized":"2024-04-05T03:03:44.122011+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:45:00.227236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121540+0000","last_change":"2024-04-05T03:02:48.461320+0000","last_active":"2024-04-05T03:03:44.121540+0000","last_peered":"2024-04-05T03:03:44.121540+0000","last_clean":"2024-04-05T03:03:44.121540+0000","last_became_active":"2024-04-05T03:02:48.461034+0000","last_became_peered":"2024-04-05T03:02:48.461034+0000","last_unstale":"2024-04-05T03:03:44.121540+0000","last_undegraded":"2024-04-05T03:03:44.121540+0000","last_fullsized":"2024-04-05T03:03:44.121540+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:34:08.199664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122756+0000","last_change":"2024-04-05T03:02:50.002729+0000","last_active":"2024-04-05T03:03:44.122756+0000","last_peered":"2024-04-05T03:03:44.122756+0000","last_clean":"2024-04-05T03:03:44.122756+0000","last_became_active":"2024-04-05T03:02:50.002342+0000","last_became_peered":"2024-04-05T03:02:50.002342+0000","last_unstale":"2024-04-05T03:03:44.122756+0000","last_undegraded":"2024-04-05T03:03:44.122756+0000","last_fullsized":"2024-04-05T03:03:44.122756+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:51:56.223511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127209+0000","last_change":"2024-04-05T03:02:48.459896+0000","last_active":"2024-04-05T03:03:44.127209+0000","last_peered":"2024-04-05T03:03:44.127209+0000","last_clean":"2024-04-05T03:03:44.127209+0000","last_became_active":"2024-04-05T03:02:48.458108+0000","last_became_peered":"2024-04-05T03:02:48.458108+0000","last_unstale":"2024-04-05T03:03:44.127209+0000","last_undegraded":"2024-04-05T03:03:44.127209+0000","last_fullsized":"2024-04-05T03:03:44.127209+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:41:12.902400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126553+0000","last_change":"2024-04-05T03:02:48.903143+0000","last_active":"2024-04-05T03:03:44.126553+0000","last_peered":"2024-04-05T03:03:44.126553+0000","last_clean":"2024-04-05T03:03:44.126553+0000","last_became_active":"2024-04-05T03:02:48.902265+0000","last_became_peered":"2024-04-05T03:02:48.902265+0000","last_unstale":"2024-04-05T03:03:44.126553+0000","last_undegraded":"2024-04-05T03:03:44.126553+0000","last_fullsized":"2024-04-05T03:03:44.126553+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:05:56.179576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.126604+0000","last_change":"2024-04-05T03:02:48.455833+0000","last_active":"2024-04-05T03:03:44.126604+0000","last_peered":"2024-04-05T03:03:44.126604+0000","last_clean":"2024-04-05T03:03:44.126604+0000","last_became_active":"2024-04-05T03:02:48.455185+0000","last_became_peered":"2024-04-05T03:02:48.455185+0000","last_unstale":"2024-04-05T03:03:44.126604+0000","last_undegraded":"2024-04-05T03:03:44.126604+0000","last_fullsized":"2024-04-05T03:03:44.126604+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:02.621579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.124971+0000","last_change":"2024-04-05T03:02:49.746606+0000","last_active":"2024-04-05T03:03:44.124971+0000","last_peered":"2024-04-05T03:03:44.124971+0000","last_clean":"2024-04-05T03:03:44.124971+0000","last_became_active":"2024-04-05T03:02:49.746308+0000","last_became_peered":"2024-04-05T03:02:49.746308+0000","last_unstale":"2024-04-05T03:03:44.124971+0000","last_undegraded":"2024-04-05T03:03:44.124971+0000","last_fullsized":"2024-04-05T03:03:44.124971+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:07:39.965218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.127576+0000","last_change":"2024-04-05T03:02:48.456697+0000","last_active":"2024-04-05T03:03:44.127576+0000","last_peered":"2024-04-05T03:03:44.127576+0000","last_clean":"2024-04-05T03:03:44.127576+0000","last_became_active":"2024-04-05T03:02:48.456140+0000","last_became_peered":"2024-04-05T03:02:48.456140+0000","last_unstale":"2024-04-05T03:03:44.127576+0000","last_undegraded":"2024-04-05T03:03:44.127576+0000","last_fullsized":"2024-04-05T03:03:44.127576+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:57:31.290707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122585+0000","last_change":"2024-04-05T03:02:50.002658+0000","last_active":"2024-04-05T03:03:44.122585+0000","last_peered":"2024-04-05T03:03:44.122585+0000","last_clean":"2024-04-05T03:03:44.122585+0000","last_became_active":"2024-04-05T03:02:50.002383+0000","last_became_peered":"2024-04-05T03:02:50.002383+0000","last_unstale":"2024-04-05T03:03:44.122585+0000","last_undegraded":"2024-04-05T03:03:44.122585+0000","last_fullsized":"2024-04-05T03:03:44.122585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:05:34.117134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121968+0000","last_change":"2024-04-05T03:02:48.457005+0000","last_active":"2024-04-05T03:03:44.121968+0000","last_peered":"2024-04-05T03:03:44.121968+0000","last_clean":"2024-04-05T03:03:44.121968+0000","last_became_active":"2024-04-05T03:02:48.456670+0000","last_became_peered":"2024-04-05T03:02:48.456670+0000","last_unstale":"2024-04-05T03:03:44.121968+0000","last_undegraded":"2024-04-05T03:03:44.121968+0000","last_fullsized":"2024-04-05T03:03:44.121968+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:45:12.114782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122420+0000","last_change":"2024-04-05T03:02:49.471218+0000","last_active":"2024-04-05T03:03:44.122420+0000","last_peered":"2024-04-05T03:03:44.122420+0000","last_clean":"2024-04-05T03:03:44.122420+0000","last_became_active":"2024-04-05T03:02:49.470918+0000","last_became_peered":"2024-04-05T03:02:49.470918+0000","last_unstale":"2024-04-05T03:03:44.122420+0000","last_undegraded":"2024-04-05T03:03:44.122420+0000","last_fullsized":"2024-04-05T03:03:44.122420+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:48:19.807934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122364+0000","last_change":"2024-04-05T03:02:48.463560+0000","last_active":"2024-04-05T03:03:44.122364+0000","last_peered":"2024-04-05T03:03:44.122364+0000","last_clean":"2024-04-05T03:03:44.122364+0000","last_became_active":"2024-04-05T03:02:48.463245+0000","last_became_peered":"2024-04-05T03:02:48.463245+0000","last_unstale":"2024-04-05T03:03:44.122364+0000","last_undegraded":"2024-04-05T03:03:44.122364+0000","last_fullsized":"2024-04-05T03:03:44.122364+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:47:16.914625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.134193+0000","last_change":"2024-04-05T03:03:45.134193+0000","last_active":"2024-04-05T03:03:45.134193+0000","last_peered":"2024-04-05T03:03:45.134193+0000","last_clean":"2024-04-05T03:03:45.134193+0000","last_became_active":"2024-04-05T03:03:45.133891+0000","last_became_peered":"2024-04-05T03:03:45.133891+0000","last_unstale":"2024-04-05T03:03:45.134193+0000","last_undegraded":"2024-04-05T03:03:45.134193+0000","last_fullsized":"2024-04-05T03:03:45.134193+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:56:06.527860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.122675+0000","last_change":"2024-04-05T03:02:48.457628+0000","last_active":"2024-04-05T03:03:44.122675+0000","last_peered":"2024-04-05T03:03:44.122675+0000","last_clean":"2024-04-05T03:03:44.122675+0000","last_became_active":"2024-04-05T03:02:48.457318+0000","last_became_peered":"2024-04-05T03:02:48.457318+0000","last_unstale":"2024-04-05T03:03:44.122675+0000","last_undegraded":"2024-04-05T03:03:44.122675+0000","last_fullsized":"2024-04-05T03:03:44.122675+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:32.231625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.121893+0000","last_change":"2024-04-05T03:02:50.004105+0000","last_active":"2024-04-05T03:03:44.121893+0000","last_peered":"2024-04-05T03:03:44.121893+0000","last_clean":"2024-04-05T03:03:44.121893+0000","last_became_active":"2024-04-05T03:02:50.003693+0000","last_became_peered":"2024-04-05T03:02:50.003693+0000","last_unstale":"2024-04-05T03:03:44.121893+0000","last_undegraded":"2024-04-05T03:03:44.121893+0000","last_fullsized":"2024-04-05T03:03:44.121893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:39:02.044564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125825+0000","last_change":"2024-04-05T03:02:48.464554+0000","last_active":"2024-04-05T03:03:44.125825+0000","last_peered":"2024-04-05T03:03:44.125825+0000","last_clean":"2024-04-05T03:03:44.125825+0000","last_became_active":"2024-04-05T03:02:48.463923+0000","last_became_peered":"2024-04-05T03:02:48.463923+0000","last_unstale":"2024-04-05T03:03:44.125825+0000","last_undegraded":"2024-04-05T03:03:44.125825+0000","last_fullsized":"2024-04-05T03:03:44.125825+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:56.699626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T03:03:45.256204+0000","last_change":"2024-04-05T03:03:45.256204+0000","last_active":"2024-04-05T03:03:45.256204+0000","last_peered":"2024-04-05T03:03:45.256204+0000","last_clean":"2024-04-05T03:03:45.256204+0000","last_became_active":"2024-04-05T03:03:45.255812+0000","last_became_peered":"2024-04-05T03:03:45.255812+0000","last_unstale":"2024-04-05T03:03:45.256204+0000","last_undegraded":"2024-04-05T03:03:45.256204+0000","last_fullsized":"2024-04-05T03:03:45.256204+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:15:31.145449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T03:03:44.125800+0000","last_change":"2024-04-05T03:02:48.456415+0000","last_active":"2024-04-05T03:03:44.125800+0000","last_peered":"2024-04-05T03:03:44.125800+0000","last_clean":"2024-04-05T03:03:44.125800+0000","last_became_active":"2024-04-05T03:02:48.455959+0000","last_became_peered":"2024-04-05T03:02:48.455959+0000","last_unstale":"2024-04-05T03:03:44.125800+0000","last_undegraded":"2024-04-05T03:03:44.125800+0000","last_fullsized":"2024-04-05T03:03:44.125800+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:52:33.266219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298660,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446492,"statfs":{"total":95995035648,"available":95689207808,"internally_reserved":0,"allocated":1335296,"data_stored":902348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.38400000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51000000000000001,"15min":0.50800000000000001},"min":{"1min":0.38100000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.69499999999999995}]},{"osd":1,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78700000000000003,"5min":0.878,"15min":0.878},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.379,"5min":0.371,"15min":0.371},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60499999999999998}]},{"osd":2,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.50700000000000001,"15min":0.505},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.56200000000000006}]},{"osd":3,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55400000000000005,"15min":0.55300000000000005},"min":{"1min":0.42299999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72999999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.625}]},{"osd":4,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48199999999999998,"15min":0.48399999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.49399999999999999,"15min":0.5},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.70099999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.73099999999999998}]},{"osd":5,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.502,"15min":0.50600000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.77400000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.77100000000000002,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.75900000000000001}]},{"osd":6,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.39800000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77000000000000002,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.432,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78800000000000003,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.77600000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298644,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446508,"statfs":{"total":95995035648,"available":95689224192,"internally_reserved":0,"allocated":1318912,"data_stored":901958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.71099999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.70899999999999996,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.34899999999999998}]},{"osd":1,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.39200000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.39300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.55900000000000005}]},{"osd":2,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.46200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69399999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.39300000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.64900000000000002,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.51500000000000001}]},{"osd":3,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57299999999999995,"15min":0.56000000000000005},"min":{"1min":0.438,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.433}]},{"osd":4,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.44,"15min":0.42999999999999999},"min":{"1min":0.315,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.45100000000000001,"15min":0.42699999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59699999999999998}]},{"osd":5,"last update":"Fri Apr 5 03:03:49 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.46400000000000002,"15min":0.44700000000000001},"min":{"1min":0.36499999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.626,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.52600000000000002,"15min":0.499},"min":{"1min":0.40400000000000003,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.67800000000000005}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.64000000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":708608,"data_stored":309947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.35399999999999998,"15min":0.316},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.56599999999999995,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.374},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.39400000000000002,"15min":0.34999999999999998},"min":{"1min":0.27100000000000002,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.59699999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.32000000000000001}]},{"osd":2,"last update":"Fri Apr 5 03:03:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40600000000000003,"15min":0.39400000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.621,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.39200000000000002,"15min":0.37},"min":{"1min":0.24199999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.53800000000000003,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":0.39000000000000001}]},{"osd":3,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41099999999999998,"15min":0.41899999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.60599999999999998,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.46800000000000003,"15min":0.46400000000000002},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60699999999999998,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.40300000000000002}]},{"osd":4,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63200000000000001,"15min":0.63900000000000001},"min":{"1min":0.45600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.85399999999999998,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61699999999999999,"15min":0.61799999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86199999999999999,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Apr 5 03:03:29 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64100000000000001,"15min":0.64300000000000002},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93999999999999995,"5min":1.7709999999999999,"15min":1.7709999999999999},"last":0.50800000000000001}]},{"osd":6,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.63400000000000001,"15min":0.623},"min":{"1min":0.375,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0029999999999999,"5min":1.74,"15min":1.74},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.629,"15min":0.63600000000000001},"min":{"1min":0.45400000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Fri Apr 5 03:03:13 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59999999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.659,"5min":1.659,"15min":1.659},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.61199999999999999,"15min":0.60899999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.61699999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1314816,"data_stored":901172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 03:03:57 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36499999999999999,"15min":0.32700000000000001},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.186},"max":{"1min":0.623,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.38500000000000001,"15min":0.34399999999999997},"min":{"1min":0.249,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.58999999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.49399999999999999}]},{"osd":2,"last update":"Fri Apr 5 03:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.46300000000000002,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.54000000000000004,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.44400000000000001}]},{"osd":3,"last update":"Fri Apr 5 03:03:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.441,"15min":0.44400000000000001},"min":{"1min":0.32300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.61299999999999999,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.50600000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.52000000000000002}]},{"osd":4,"last update":"Fri Apr 5 03:03:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78500000000000003,"5min":0.875,"15min":0.875},"last":0.59299999999999997}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.41699999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79500000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83599999999999997,"5min":1.018,"15min":1.018},"last":0.64300000000000002}]},{"osd":6,"last update":"Fri Apr 5 03:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60599999999999998,"15min":0.60799999999999998},"min":{"1min":0.438,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81399999999999995,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.82399999999999995,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.67800000000000005}]},{"osd":7,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.63200000000000001,"15min":0.63500000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74099999999999999,"5min":0.875,"15min":0.875},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.69699999999999995,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80600000000000005,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.55900000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":310816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.38600000000000001},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51000000000000001,"5min":0.623,"15min":0.623},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.39000000000000001,"15min":0.39400000000000002},"min":{"1min":0.25900000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.53000000000000003,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.26600000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.39500000000000002,"15min":0.40500000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57599999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.27600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.54200000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.307}]},{"osd":3,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.44500000000000001,"15min":0.45900000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.55800000000000005,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.43099999999999999,"15min":0.45100000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.52900000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.375}]},{"osd":4,"last update":"Fri Apr 5 03:03:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.41299999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.83699999999999997,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.41499999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86799999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.65500000000000003}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80800000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.0649999999999999,"5min":0.66600000000000004,"15min":0.60899999999999999},"min":{"1min":0.42599999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":12.968999999999999,"5min":12.968999999999999,"15min":12.968999999999999},"last":0.67500000000000004}]},{"osd":6,"last update":"Fri Apr 5 03:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.59999999999999998,"15min":0.59699999999999998},"min":{"1min":0.45900000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.62,"15min":0.624},"min":{"1min":0.46899999999999997,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.628}]},{"osd":7,"last update":"Fri Apr 5 03:03:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76500000000000001,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.626,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.60399999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":310302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39100000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.433,"15min":0.443},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81499999999999995,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.32600000000000001}]},{"osd":1,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.45200000000000001,"15min":0.44900000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.85999999999999999,"5min":1.54,"15min":1.54},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44900000000000001,"15min":0.45700000000000002},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":1.468,"15min":1.468},"last":0.34399999999999997}]},{"osd":2,"last update":"Fri Apr 5 03:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.48299999999999998,"15min":0.496},"min":{"1min":0.28399999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.94899999999999995,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.31},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.22800000000000001}]},{"osd":4,"last update":"Fri Apr 5 03:03:03 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.434,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.64200000000000002,"15min":0.65700000000000003},"min":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.96899999999999997,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.50900000000000001}]},{"osd":5,"last update":"Fri Apr 5 03:03:25 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.432,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.98599999999999999,"5min":1.002,"15min":1.002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63600000000000001,"15min":0.64300000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.60699999999999998}]},{"osd":6,"last update":"Fri Apr 5 03:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.629,"5min":0.64900000000000002,"15min":0.65600000000000003},"min":{"1min":0.45500000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89300000000000002,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.71899999999999997}]},{"osd":7,"last update":"Fri Apr 5 03:03:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.63700000000000001,"15min":0.64100000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.629},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46800000000000003,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.57699999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":737280,"data_stored":311910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.56100000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54700000000000004,"15min":0.56000000000000005},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66700000000000004,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.66900000000000004}]},{"osd":1,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.753,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.55200000000000005,"15min":0.56200000000000006},"min":{"1min":0.34499999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.67700000000000005,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.66000000000000003}]},{"osd":2,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.64600000000000002,"15min":0.57299999999999995},"min":{"1min":0.372,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":11.239000000000001,"5min":11.239000000000001,"15min":11.239000000000001},"last":0.63},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.57199999999999995,"15min":0.58999999999999997},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.69499999999999995,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56200000000000006}]},{"osd":3,"last update":"Fri Apr 5 03:03:04 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.67900000000000005,"15min":0.61499999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":11.209,"5min":11.209,"15min":11.209},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.133,"5min":0.69099999999999995,"15min":0.625},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.221,"5min":11.221,"15min":11.221},"last":0.69599999999999995}]},{"osd":5,"last update":"Fri Apr 5 03:03:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.40200000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59599999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.432},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.623,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.64500000000000002}]},{"osd":6,"last update":"Fri Apr 5 03:03:58 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.46700000000000003,"15min":0.45100000000000001},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.63800000000000001,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.47499999999999998,"15min":0.45500000000000002},"min":{"1min":0.35099999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.73799999999999999}]},{"osd":7,"last update":"Fri Apr 5 03:03:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.49299999999999999,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.58599999999999997,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.438,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.35899999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.56999999999999995,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.68500000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":720896,"data_stored":310852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74399999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.504},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.627}]},{"osd":1,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.52300000000000002},"min":{"1min":0.39600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.78400000000000003,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.63700000000000001,"15min":0.58099999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.2070000000000007,"5min":9.2070000000000007,"15min":9.2070000000000007},"last":0.57599999999999996}]},{"osd":2,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.443,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.55100000000000005,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.81599999999999995,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.60099999999999998}]},{"osd":3,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.75,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.56699999999999995,"15min":0.55700000000000005},"min":{"1min":0.437,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78100000000000003,"5min":1.329,"15min":1.329},"last":0.61299999999999999}]},{"osd":4,"last update":"Fri Apr 5 03:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.40899999999999997,"15min":0.38100000000000001},"min":{"1min":0.29799999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.42999999999999999,"15min":0.40200000000000002},"min":{"1min":0.32300000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.71499999999999997,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56299999999999994}]},{"osd":6,"last update":"Fri Apr 5 03:03:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48699999999999999,"15min":0.46500000000000002},"min":{"1min":0.38,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.443},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.5,"15min":0.47199999999999998},"min":{"1min":0.38900000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.81499999999999995,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.40600000000000003}]},{"osd":7,"last update":"Fri Apr 5 03:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81999999999999995,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77000000000000002,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.53400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:04:04.966 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-05T03:04:04.966 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-05T03:04:04.966 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-05T03:04:04.967 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:04:05.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:05 smithi033 ceph-mon[29244]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:05.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:05 smithi033 ceph-mon[40008]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:05.237 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:05.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:05 smithi151 ceph-mon[32789]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:06.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:06 smithi033 ceph-mon[29244]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:04:06.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:06 smithi033 ceph-mon[40008]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:04:06.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:06 smithi151 ceph-mon[32789]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:04:07.016 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:04:07.017 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-05T03:04:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:07 smithi151 ceph-mon[32789]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:07 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3765142797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:04:07.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:04:07 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:04:07.320 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:07 smithi033 ceph-mon[29244]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:07.320 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:07 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3765142797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:04:07.321 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:07 smithi033 ceph-mon[40008]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:07.321 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:07 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3765142797' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:04:07.700 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-05T03:04:07.700 INFO:teuthology.run_tasks:Running task print... 2024-04-05T03:04:07.711 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-05T03:04:07.711 INFO:teuthology.run_tasks:Running task parallel... 2024-04-05T03:04:07.734 INFO:teuthology.task.parallel:starting parallel... 2024-04-05T03:04:07.735 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-05T03:04:07.735 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-05T03:04:07.739 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-04-05T03:04:07.740 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-05T03:04:07.740 DEBUG:teuthology.orchestra.run.smithi033:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-05T03:04:07.740 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-05T03:04:07.741 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-05T03:04:07.741 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi033.front.sepia.ceph.com 2024-04-05T03:04:07.741 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-05T03:04:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:04:07.758 INFO:teuthology.orchestra.run.smithi033.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-04-05T03:04:07.758 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-04-05T03:04:07.813 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-04-05T03:04:07.814 DEBUG:teuthology.orchestra.run.smithi033:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-04-05T03:04:07.868 INFO:tasks.workunit:timeout=3h 2024-04-05T03:04:07.868 INFO:tasks.workunit:cleanup=True 2024-04-05T03:04:07.869 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-04-05T03:04:07.923 INFO:tasks.workunit.client.0.smithi033.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-05T03:04:08.011 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:09.410 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-mon[29244]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:09.410 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-mon[40008]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:09.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:09 smithi151 ceph-mon[32789]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:10.169 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:04:09] ENGINE Bus STOPPING 2024-04-05T03:04:10.169 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:04:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:04:10.169 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:04:09] ENGINE Bus STOPPED 2024-04-05T03:04:10.169 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:04:09] ENGINE Bus STARTING 2024-04-05T03:04:10.170 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:04:09] ENGINE Serving on http://:::9283 2024-04-05T03:04:10.170 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:04:09] ENGINE Bus STARTED 2024-04-05T03:04:10.545 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-05T03:04:10.854 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:11.198 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[40008]: from='client.? ' entity='client.admin' 2024-04-05T03:04:11.198 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:11.198 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[40008]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:11.198 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[29244]: from='client.? ' entity='client.admin' 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[29244]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:11.199 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:10 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:11.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:10 smithi151 ceph-mon[32789]: from='client.? ' entity='client.admin' 2024-04-05T03:04:11.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:10 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:11.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:10 smithi151 ceph-mon[32789]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:11.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:10 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:11.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:10 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:11.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:10 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:12.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:12] "GET /metrics HTTP/1.1" 200 239239 "" "Prometheus/2.43.0" 2024-04-05T03:04:13.477 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:13 smithi033 ceph-mon[40008]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:13.477 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:13 smithi033 ceph-mon[29244]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:13.505 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-05T03:04:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:13 smithi151 ceph-mon[32789]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:13.771 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:14.155 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:14 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:14.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:14 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:14.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:14 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:15.378 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:15 smithi033 ceph-mon[29244]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:15.378 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:15 smithi033 ceph-mon[40008]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:15 smithi151 ceph-mon[32789]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:15.620 INFO:teuthology.orchestra.run.smithi033.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T03:04:16.487 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:04:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:04:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T03:04:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[40008]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:16.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:16.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:16.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T03:04:16.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:16 smithi033 ceph-mon[29244]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:16.762 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:17.023 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T03:04:17.024 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:16 smithi151 ceph-mon[32789]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:17.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:04:17 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (86s) 27s ago 2m 24.5M - 0.25.0 c8568f914cd2 fc398a28fe81 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi151 *:3000 running (113s) 31s ago 112s 81.2M - 9.4.7 954c08fa6188 a5ab46c212de 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi033.rmzwzs smithi033 running (52s) 27s ago 50s 12.5M - 17.2.7-805-g6fda41ff a49113eacc58 c7d056f2104c 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi151.cxpowy smithi151 running (48s) 31s ago 47s 14.2M - 17.2.7-805-g6fda41ff a49113eacc58 0d112d294451 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi151 *:8443 running (6m) 31s ago 6m 432M - 17.2.7-805-g6fda41ff a49113eacc58 c743b76f8587 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:9283 running (8m) 27s ago 8m 477M - 17.2.7-805-g6fda41ff a49113eacc58 46497f82f211 2024-04-05T03:04:18.920 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (8m) 27s ago 8m 42.7M 2048M 17.2.7-805-g6fda41ff a49113eacc58 e876c4ff5784 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi151 running (6m) 31s ago 6m 39.9M 2048M 17.2.7-805-g6fda41ff a49113eacc58 464b19e4a343 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (6m) 27s ago 6m 38.3M 2048M 17.2.7-805-g6fda41ff a49113eacc58 a5676722d2cb 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (104s) 27s ago 103s 15.9M - 1.5.0 0da6a335fe13 94bec9c472c2 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi151 *:9100 running (100s) 31s ago 100s 8288k - 1.5.0 0da6a335fe13 958473fd6ad3 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (6m) 27s ago 5m 42.5M 4017M 17.2.7-805-g6fda41ff a49113eacc58 d0c1cb9b20a4 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (5m) 27s ago 5m 43.4M 4017M 17.2.7-805-g6fda41ff a49113eacc58 279958cf423a 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (5m) 27s ago 5m 39.3M 4017M 17.2.7-805-g6fda41ff a49113eacc58 309a3fe45561 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (4m) 27s ago 4m 40.4M 4017M 17.2.7-805-g6fda41ff a49113eacc58 2b3b4d99eb65 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi151 running (4m) 31s ago 4m 40.9M 4016M 17.2.7-805-g6fda41ff a49113eacc58 b7f73016d7ed 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi151 running (4m) 31s ago 4m 43.5M 4016M 17.2.7-805-g6fda41ff a49113eacc58 e2607db1cdd0 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi151 running (3m) 31s ago 3m 42.5M 4016M 17.2.7-805-g6fda41ff a49113eacc58 2063f47e19f2 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi151 running (3m) 31s ago 3m 41.0M 4016M 17.2.7-805-g6fda41ff a49113eacc58 164624ba7a6d 2024-04-05T03:04:18.921 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi151 *:9095 running (80s) 31s ago 2m 40.5M - 2.43.0 a07b618ecd1d c19cfcd8e648 2024-04-05T03:04:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:19 smithi033 ceph-mon[29244]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:19.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:19 smithi033 ceph-mon[40008]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:19.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:19 smithi151 ceph-mon[32789]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:19.977 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-05T03:04:20.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:20 smithi033 ceph-mon[29244]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:04:20.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:20 smithi033 ceph-mon[40008]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:04:20.299 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:20.398 DEBUG:teuthology.orchestra.run.smithi033:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-05T03:04:20.432 INFO:tasks.workunit.client.0.smithi033.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-05T03:04:20.434 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-05T03:04:20.434 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-05T03:04:20.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:20 smithi151 ceph-mon[32789]: from='client.24743 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:04:21.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:21 smithi033 ceph-mon[40008]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:21 smithi033 ceph-mon[29244]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:21.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:21 smithi151 ceph-mon[32789]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:22.336 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:22 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:22] "GET /metrics HTTP/1.1" 200 239229 "" "Prometheus/2.43.0" 2024-04-05T03:04:22.460 INFO:teuthology.orchestra.run.smithi033.stdout:wait for minority of mons to upgrade 2024-04-05T03:04:22.816 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-05T03:04:22.933 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-05T03:04:22.974 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-05T03:04:22.976 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-05T03:04:22.976 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-05T03:04:23.015 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-05T03:04:23.019 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-04-05T03:04:23.019 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-05T03:04:23.075 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-04-05T03:04:23.076 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-04-05T03:04:23.076 DEBUG:teuthology.orchestra.run.smithi033:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-04-05T03:04:23.135 INFO:tasks.workunit.client.0.smithi033.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-04-05T03:04:23.135 INFO:tasks.workunit.client.0.smithi033.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-04-05T03:04:23.135 INFO:tasks.workunit.client.0.smithi033.stderr:+ TMPDIR=/tmp/rbd_import_export_81330 2024-04-05T03:04:23.135 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm -rf /tmp/rbd_import_export_81330 2024-04-05T03:04:23.135 INFO:tasks.workunit.client.0.smithi033.stderr:+ mkdir /tmp/rbd_import_export_81330 2024-04-05T03:04:23.136 INFO:tasks.workunit.client.0.smithi033.stderr:+ trap 'rm -rf /tmp/rbd_import_export_81330' INT TERM EXIT 2024-04-05T03:04:23.137 INFO:tasks.workunit.client.0.smithi033.stderr:+ mkdir foo.81330 2024-04-05T03:04:23.137 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import foo.81330 foo.dir 2024-04-05T03:04:23.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:23 smithi033 ceph-mon[29244]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:23.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:23 smithi033 ceph-mon[40008]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:23.255 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: cannot import a directory 2024-04-05T03:04:23.256 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 0% complete...failed. 2024-04-05T03:04:23.256 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: import failed: (21) Is a directory 2024-04-05T03:04:23.269 INFO:tasks.workunit.client.0.smithi033.stderr:+ true 2024-04-05T03:04:23.269 INFO:tasks.workunit.client.0.smithi033.stderr:+ rmdir foo.81330 2024-04-05T03:04:23.311 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_81330/img bs=1k count=1 seek=10 2024-04-05T03:04:23.311 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records in 2024-04-05T03:04:23.311 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records out 2024-04-05T03:04:23.311 INFO:tasks.workunit.client.0.smithi033.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.2079e-05 s, 24.3 MB/s 2024-04-05T03:04:23.311 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_81330/img bs=1k count=10 seek=100 2024-04-05T03:04:23.312 INFO:tasks.workunit.client.0.smithi033.stderr:10+0 records in 2024-04-05T03:04:23.312 INFO:tasks.workunit.client.0.smithi033.stderr:10+0 records out 2024-04-05T03:04:23.312 INFO:tasks.workunit.client.0.smithi033.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.184e-05 s, 198 MB/s 2024-04-05T03:04:23.312 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_81330/img bs=1k count=100 seek=1000 2024-04-05T03:04:23.313 INFO:tasks.workunit.client.0.smithi033.stderr:60+1 records in 2024-04-05T03:04:23.313 INFO:tasks.workunit.client.0.smithi033.stderr:60+1 records out 2024-04-05T03:04:23.313 INFO:tasks.workunit.client.0.smithi033.stderr:61464 bytes (61 kB, 60 KiB) copied, 0.000166214 s, 370 MB/s 2024-04-05T03:04:23.313 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_81330/img bs=1k seek=10000 2024-04-05T03:04:23.314 INFO:tasks.workunit.client.0.smithi033.stderr:137+1 records in 2024-04-05T03:04:23.314 INFO:tasks.workunit.client.0.smithi033.stderr:137+1 records out 2024-04-05T03:04:23.314 INFO:tasks.workunit.client.0.smithi033.stderr:140760 bytes (141 kB, 137 KiB) copied, 0.000323673 s, 435 MB/s 2024-04-05T03:04:23.314 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_81330/img bs=1k seek=100000 2024-04-05T03:04:23.314 INFO:tasks.workunit.client.0.smithi033.stderr:59+1 records in 2024-04-05T03:04:23.315 INFO:tasks.workunit.client.0.smithi033.stderr:59+1 records out 2024-04-05T03:04:23.315 INFO:tasks.workunit.client.0.smithi033.stderr:61408 bytes (61 kB, 60 KiB) copied, 0.000154236 s, 398 MB/s 2024-04-05T03:04:23.315 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_81330/img bs=1k seek=1000000 2024-04-05T03:04:23.315 INFO:tasks.workunit.client.0.smithi033.stderr:154+1 records in 2024-04-05T03:04:23.315 INFO:tasks.workunit.client.0.smithi033.stderr:154+1 records out 2024-04-05T03:04:23.315 INFO:tasks.workunit.client.0.smithi033.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.000359564 s, 440 MB/s 2024-04-05T03:04:23.316 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm testimg 2024-04-05T03:04:23.354 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 0% complete...failed. 2024-04-05T03:04:23.354 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: delete error: (2) No such file or directory 2024-04-05T03:04:23.357 INFO:tasks.workunit.client.0.smithi033.stderr:+ true 2024-04-05T03:04:23.357 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format /tmp/rbd_import_export_81330/img testimg 2024-04-05T03:04:23.403 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-05T03:04:23.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:23 smithi151 ceph-mon[32789]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:23.731 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:25.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:25 smithi033 ceph-mon[29244]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:25.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:25 smithi033 ceph-mon[29244]: osdmap e71: 8 total, 8 up, 8 in 2024-04-05T03:04:25.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:25 smithi033 ceph-mon[40008]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:25.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:25 smithi033 ceph-mon[40008]: osdmap e71: 8 total, 8 up, 8 in 2024-04-05T03:04:25.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:25 smithi151 ceph-mon[32789]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:25.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:25 smithi151 ceph-mon[32789]: osdmap e71: 8 total, 8 up, 8 in 2024-04-05T03:04:25.673 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-04-05T03:04:25.674 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:25.677 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export testimg /tmp/rbd_import_export_81330/img2 2024-04-05T03:04:25.969 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:25.972 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export testimg - 2024-04-05T03:04:26.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:26 smithi033 ceph-mon[29244]: osdmap e72: 8 total, 8 up, 8 in 2024-04-05T03:04:26.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:26 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1874585610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:26 smithi033 ceph-mon[40008]: osdmap e72: 8 total, 8 up, 8 in 2024-04-05T03:04:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:26 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1874585610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:26 smithi151 ceph-mon[32789]: osdmap e72: 8 total, 8 up, 8 in 2024-04-05T03:04:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:26 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1874585610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:26.726 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:26.729 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm testimg 2024-04-05T03:04:26.818 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-04-05T03:04:26.820 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp /tmp/rbd_import_export_81330/img /tmp/rbd_import_export_81330/img2 2024-04-05T03:04:27.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:27 smithi151 ceph-mon[32789]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-04-05T03:04:27.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:04:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:04:27.387 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp /tmp/rbd_import_export_81330/img /tmp/rbd_import_export_81330/img3 2024-04-05T03:04:27.401 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:27 smithi033 ceph-mon[29244]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-04-05T03:04:27.401 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:27 smithi033 ceph-mon[40008]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-04-05T03:04:27.785 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img2 /tmp/rbd_import_export_81330/img3 2024-04-05T03:04:27.964 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format - testimg 2024-04-05T03:04:28.364 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:28.364 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:28.367 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export testimg /tmp/rbd_import_export_81330/img2 2024-04-05T03:04:28.651 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:28.654 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export testimg - 2024-04-05T03:04:29.445 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:29.448 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm testimg 2024-04-05T03:04:29.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:29 smithi033 ceph-mon[29244]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-04-05T03:04:29.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:29 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1570672332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:29.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:29 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:29.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:29 smithi033 ceph-mon[40008]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-04-05T03:04:29.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:29 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1570672332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:29.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:29 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:29 smithi151 ceph-mon[32789]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2024-04-05T03:04:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:29 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1570672332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:29 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:29.536 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-04-05T03:04:29.539 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp /tmp/rbd_import_export_81330/img /tmp/rbd_import_export_81330/img2 2024-04-05T03:04:30.115 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp /tmp/rbd_import_export_81330/img /tmp/rbd_import_export_81330/img3 2024-04-05T03:04:30.530 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img /tmp/rbd_import_export_81330/img2 /tmp/rbd_import_export_81330/img3 2024-04-05T03:04:30.768 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd help export 2024-04-05T03:04:30.768 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep -q export-format 2024-04-05T03:04:30.786 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_81330/img bs=1k count=10 seek=100 2024-04-05T03:04:30.787 INFO:tasks.workunit.client.0.smithi033.stderr:10+0 records in 2024-04-05T03:04:30.787 INFO:tasks.workunit.client.0.smithi033.stderr:10+0 records out 2024-04-05T03:04:30.787 INFO:tasks.workunit.client.0.smithi033.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.824e-05 s, 176 MB/s 2024-04-05T03:04:30.787 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format /tmp/rbd_import_export_81330/img testimg 2024-04-05T03:04:30.848 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:30.849 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:30.851 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap create testimg@snap 2024-04-05T03:04:31.168 INFO:tasks.workunit.client.0.smithi033.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-05T03:04:31.174 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd image-meta set testimg key1 value1 2024-04-05T03:04:31.224 INFO:tasks.workunit.client.0.smithi033.stderr:++ rbd image-meta list testimg 2024-04-05T03:04:31.263 INFO:tasks.workunit.client.0.smithi033.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-04-05T03:04:31.263 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-04-05T03:04:31.263 INFO:tasks.workunit.client.0.smithi033.stderr:Key Value 2024-04-05T03:04:31.263 INFO:tasks.workunit.client.0.smithi033.stderr:key1 value1' 2024-04-05T03:04:31.263 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:31.302 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:31.305 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_81330/img_v2 testimg_import 2024-04-05T03:04:31.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:31 smithi033 ceph-mon[29244]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 KiB/s rd, 116 KiB/s wr, 100 op/s 2024-04-05T03:04:31.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:31 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/501182922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:31.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:31 smithi033 ceph-mon[40008]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 KiB/s rd, 116 KiB/s wr, 100 op/s 2024-04-05T03:04:31.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:31 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/501182922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:31 smithi151 ceph-mon[32789]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 KiB/s rd, 116 KiB/s wr, 100 op/s 2024-04-05T03:04:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:31 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/501182922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:32.188 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-05T03:04:32.192 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd info testimg_import 2024-04-05T03:04:32.227 INFO:tasks.workunit.client.0.smithi033.stdout:rbd image 'testimg_import': 2024-04-05T03:04:32.227 INFO:tasks.workunit.client.0.smithi033.stdout: size 110 KiB in 1 objects 2024-04-05T03:04:32.227 INFO:tasks.workunit.client.0.smithi033.stdout: order 22 (4 MiB objects) 2024-04-05T03:04:32.227 INFO:tasks.workunit.client.0.smithi033.stdout: snapshot_count: 1 2024-04-05T03:04:32.227 INFO:tasks.workunit.client.0.smithi033.stdout: id: 3a6231109859 2024-04-05T03:04:32.227 INFO:tasks.workunit.client.0.smithi033.stdout: block_name_prefix: rbd_data.3a6231109859 2024-04-05T03:04:32.227 INFO:tasks.workunit.client.0.smithi033.stdout: format: 2 2024-04-05T03:04:32.228 INFO:tasks.workunit.client.0.smithi033.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-04-05T03:04:32.228 INFO:tasks.workunit.client.0.smithi033.stdout: op_features: 2024-04-05T03:04:32.228 INFO:tasks.workunit.client.0.smithi033.stdout: flags: 2024-04-05T03:04:32.228 INFO:tasks.workunit.client.0.smithi033.stdout: create_timestamp: Fri Apr 5 03:04:31 2024 2024-04-05T03:04:32.228 INFO:tasks.workunit.client.0.smithi033.stdout: access_timestamp: Fri Apr 5 03:04:31 2024 2024-04-05T03:04:32.228 INFO:tasks.workunit.client.0.smithi033.stdout: modify_timestamp: Fri Apr 5 03:04:31 2024 2024-04-05T03:04:32.230 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd info testimg_import@snap 2024-04-05T03:04:32.268 INFO:tasks.workunit.client.0.smithi033.stdout:rbd image 'testimg_import': 2024-04-05T03:04:32.268 INFO:tasks.workunit.client.0.smithi033.stdout: size 110 KiB in 1 objects 2024-04-05T03:04:32.268 INFO:tasks.workunit.client.0.smithi033.stdout: order 22 (4 MiB objects) 2024-04-05T03:04:32.268 INFO:tasks.workunit.client.0.smithi033.stdout: snapshot_count: 1 2024-04-05T03:04:32.268 INFO:tasks.workunit.client.0.smithi033.stdout: id: 3a6231109859 2024-04-05T03:04:32.268 INFO:tasks.workunit.client.0.smithi033.stdout: block_name_prefix: rbd_data.3a6231109859 2024-04-05T03:04:32.268 INFO:tasks.workunit.client.0.smithi033.stdout: format: 2 2024-04-05T03:04:32.269 INFO:tasks.workunit.client.0.smithi033.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-04-05T03:04:32.269 INFO:tasks.workunit.client.0.smithi033.stdout: op_features: 2024-04-05T03:04:32.269 INFO:tasks.workunit.client.0.smithi033.stdout: flags: 2024-04-05T03:04:32.269 INFO:tasks.workunit.client.0.smithi033.stdout: create_timestamp: Fri Apr 5 03:04:31 2024 2024-04-05T03:04:32.269 INFO:tasks.workunit.client.0.smithi033.stdout: access_timestamp: Fri Apr 5 03:04:31 2024 2024-04-05T03:04:32.269 INFO:tasks.workunit.client.0.smithi033.stdout: modify_timestamp: Fri Apr 5 03:04:31 2024 2024-04-05T03:04:32.269 INFO:tasks.workunit.client.0.smithi033.stdout: protected: False 2024-04-05T03:04:32.271 INFO:tasks.workunit.client.0.smithi033.stderr:++ rbd image-meta list testimg_import 2024-04-05T03:04:32.312 INFO:tasks.workunit.client.0.smithi033.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:Key Value 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:key1 value1' 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 'There is 1 metadatum on this image: 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:Key Value 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:Key Value 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:key1 value1' ']' 2024-04-05T03:04:32.313 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export testimg_import /tmp/rbd_import_export_81330/img_import 2024-04-05T03:04:32.348 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete...done. 2024-04-05T03:04:32.351 INFO:tasks.workunit.client.0.smithi033.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_81330/img /tmp/rbd_import_export_81330/img_import 2024-04-05T03:04:32.351 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp -l /tmp/rbd_import_export_81330/img /tmp/rbd_import_export_81330/img_import 2024-04-05T03:04:32.352 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/img --format %b 2024-04-05T03:04:32.353 INFO:tasks.workunit.client.0.smithi033.stderr:+ origsize=24 2024-04-05T03:04:32.353 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/img_import --format %b 2024-04-05T03:04:32.354 INFO:tasks.workunit.client.0.smithi033.stderr:+ exportsize=224 2024-04-05T03:04:32.355 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=200 2024-04-05T03:04:32.355 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=200 2024-04-05T03:04:32.355 INFO:tasks.workunit.client.0.smithi033.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-04-05T03:04:32.355 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_81330/img_snap 2024-04-05T03:04:32.390 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete...done. 2024-04-05T03:04:32.393 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_81330/img_snap_import 2024-04-05T03:04:32.430 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete...done. 2024-04-05T03:04:32.433 INFO:tasks.workunit.client.0.smithi033.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_81330/img_snap /tmp/rbd_import_export_81330/img_snap_import 2024-04-05T03:04:32.433 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp -l /tmp/rbd_import_export_81330/img_snap /tmp/rbd_import_export_81330/img_snap_import 2024-04-05T03:04:32.434 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/img_snap --format %b 2024-04-05T03:04:32.435 INFO:tasks.workunit.client.0.smithi033.stderr:+ origsize=224 2024-04-05T03:04:32.435 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/img_snap_import --format %b 2024-04-05T03:04:32.436 INFO:tasks.workunit.client.0.smithi033.stderr:+ exportsize=224 2024-04-05T03:04:32.436 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:32.436 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:32.436 INFO:tasks.workunit.client.0.smithi033.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-05T03:04:32.436 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:32.437 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img_import 2024-04-05T03:04:32.437 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img_snap 2024-04-05T03:04:32.437 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img_snap_import 2024-04-05T03:04:32.438 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap rm testimg_import@snap 2024-04-05T03:04:32.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:32 smithi033 ceph-mon[29244]: osdmap e73: 8 total, 8 up, 8 in 2024-04-05T03:04:32.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:32 smithi033 ceph-mon[40008]: osdmap e73: 8 total, 8 up, 8 in 2024-04-05T03:04:32.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:32] "GET /metrics HTTP/1.1" 200 239278 "" "Prometheus/2.43.0" 2024-04-05T03:04:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:32 smithi151 ceph-mon[32789]: osdmap e73: 8 total, 8 up, 8 in 2024-04-05T03:04:33.179 INFO:tasks.workunit.client.0.smithi033.stderr: Removing snap: 100% complete...done. 2024-04-05T03:04:33.187 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg_import 2024-04-05T03:04:33.278 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:33.280 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap rm testimg@snap 2024-04-05T03:04:33.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:33 smithi033 ceph-mon[40008]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 291 KiB/s rd, 117 KiB/s wr, 101 op/s 2024-04-05T03:04:33.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:33 smithi033 ceph-mon[40008]: osdmap e74: 8 total, 8 up, 8 in 2024-04-05T03:04:33.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:33 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3735332296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:33.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:33 smithi033 ceph-mon[29244]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 291 KiB/s rd, 117 KiB/s wr, 101 op/s 2024-04-05T03:04:33.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:33 smithi033 ceph-mon[29244]: osdmap e74: 8 total, 8 up, 8 in 2024-04-05T03:04:33.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:33 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3735332296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:33.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:33 smithi151 ceph-mon[32789]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 291 KiB/s rd, 117 KiB/s wr, 101 op/s 2024-04-05T03:04:33.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:33 smithi151 ceph-mon[32789]: osdmap e74: 8 total, 8 up, 8 in 2024-04-05T03:04:33.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:33 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3735332296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:33.868 INFO:tasks.workunit.client.0.smithi033.stderr: Removing snap: 100% complete...done. 2024-04-05T03:04:33.876 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm testimg 2024-04-05T03:04:33.965 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:33.968 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --order 20 /tmp/rbd_import_export_81330/img testimg 2024-04-05T03:04:34.024 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:34.025 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --order is deprecated, use --object-size 2024-04-05T03:04:34.028 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:34.066 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:34.069 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_81330/img_v2 testimg_import 2024-04-05T03:04:34.127 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-05T03:04:34.131 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd info testimg_import 2024-04-05T03:04:34.131 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep order 2024-04-05T03:04:34.131 INFO:tasks.workunit.client.0.smithi033.stderr:+ awk '{print $2}' 2024-04-05T03:04:34.131 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep 20 2024-04-05T03:04:34.173 INFO:tasks.workunit.client.0.smithi033.stdout:20 2024-04-05T03:04:34.173 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:34.174 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg_import 2024-04-05T03:04:34.257 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:34.259 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg 2024-04-05T03:04:34.343 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:34.346 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_81330/img testimg 2024-04-05T03:04:34.392 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:34.396 INFO:tasks.workunit.client.0.smithi033.stderr:++ rbd info testimg 2024-04-05T03:04:34.396 INFO:tasks.workunit.client.0.smithi033.stderr:++ grep features 2024-04-05T03:04:34.434 INFO:tasks.workunit.client.0.smithi033.stderr:+ FEATURES_BEFORE=' features: layering 2024-04-05T03:04:34.434 INFO:tasks.workunit.client.0.smithi033.stderr: op_features: ' 2024-04-05T03:04:34.435 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:34.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:34 smithi033 ceph-mon[29244]: osdmap e75: 8 total, 8 up, 8 in 2024-04-05T03:04:34.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:34 smithi033 ceph-mon[29244]: osdmap e76: 8 total, 8 up, 8 in 2024-04-05T03:04:34.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:34 smithi033 ceph-mon[40008]: osdmap e75: 8 total, 8 up, 8 in 2024-04-05T03:04:34.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:34 smithi033 ceph-mon[40008]: osdmap e76: 8 total, 8 up, 8 in 2024-04-05T03:04:34.484 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:34.488 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_81330/img_v2 testimg_import 2024-04-05T03:04:34.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:34 smithi151 ceph-mon[32789]: osdmap e75: 8 total, 8 up, 8 in 2024-04-05T03:04:34.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:34 smithi151 ceph-mon[32789]: osdmap e76: 8 total, 8 up, 8 in 2024-04-05T03:04:34.540 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-05T03:04:34.544 INFO:tasks.workunit.client.0.smithi033.stderr:++ rbd info testimg_import 2024-04-05T03:04:34.544 INFO:tasks.workunit.client.0.smithi033.stderr:++ grep features 2024-04-05T03:04:34.584 INFO:tasks.workunit.client.0.smithi033.stderr:+ FEATURES_AFTER=' features: layering 2024-04-05T03:04:34.584 INFO:tasks.workunit.client.0.smithi033.stderr: op_features: ' 2024-04-05T03:04:34.584 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' ' features: layering 2024-04-05T03:04:34.584 INFO:tasks.workunit.client.0.smithi033.stderr: op_features: ' '!=' ' features: layering 2024-04-05T03:04:34.584 INFO:tasks.workunit.client.0.smithi033.stderr: op_features: ' ']' 2024-04-05T03:04:34.584 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:34.585 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg_import 2024-04-05T03:04:34.649 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:34.652 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg 2024-04-05T03:04:34.717 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:34.720 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_81330/img testimg 2024-04-05T03:04:34.778 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:34.781 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:34.819 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:34.822 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_81330/img_v2 testimg_import 2024-04-05T03:04:34.882 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-05T03:04:34.885 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd info testimg_import 2024-04-05T03:04:34.885 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep 'stripe unit' 2024-04-05T03:04:34.885 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep -Ei '(4 KiB|4096)' 2024-04-05T03:04:34.921 INFO:tasks.workunit.client.0.smithi033.stdout: stripe unit: 4 KiB 2024-04-05T03:04:34.922 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd info testimg_import 2024-04-05T03:04:34.922 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep 'stripe count' 2024-04-05T03:04:34.922 INFO:tasks.workunit.client.0.smithi033.stderr:+ awk '{print $3}' 2024-04-05T03:04:34.923 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep 1000 2024-04-05T03:04:34.958 INFO:tasks.workunit.client.0.smithi033.stdout:1000 2024-04-05T03:04:34.958 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/img_v2 2024-04-05T03:04:34.959 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg_import 2024-04-05T03:04:35.045 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-04-05T03:04:35.048 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg 2024-04-05T03:04:35.137 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-04-05T03:04:35.141 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_81330/img testimg 2024-04-05T03:04:35.200 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:35.204 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap create testimg@snap1 2024-04-05T03:04:35.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:35 smithi033 ceph-mon[29244]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:35.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:35 smithi033 ceph-mon[40008]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:35 smithi151 ceph-mon[32789]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:36.194 INFO:tasks.workunit.client.0.smithi033.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-05T03:04:36.200 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap create testimg@snap2 2024-04-05T03:04:36.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:36 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2756600390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:36.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:36 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2756600390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:36 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2756600390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:37.204 INFO:tasks.workunit.client.0.smithi033.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-05T03:04:37.210 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap protect testimg@snap2 2024-04-05T03:04:37.265 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_81330/snap_protect 2024-04-05T03:04:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:37 smithi151 ceph-mon[32789]: pgmap v103: 73 pgs: 73 active+clean; 589 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 372 KiB/s rd, 41 KiB/s wr, 486 op/s 2024-04-05T03:04:37.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:37 smithi151 ceph-mon[32789]: osdmap e77: 8 total, 8 up, 8 in 2024-04-05T03:04:37.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:04:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:04:37.310 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:37.313 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_81330/snap_protect testimg_import 2024-04-05T03:04:37.471 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:37 smithi033 ceph-mon[29244]: pgmap v103: 73 pgs: 73 active+clean; 589 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 372 KiB/s rd, 41 KiB/s wr, 486 op/s 2024-04-05T03:04:37.472 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:37 smithi033 ceph-mon[29244]: osdmap e77: 8 total, 8 up, 8 in 2024-04-05T03:04:37.472 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:37 smithi033 ceph-mon[40008]: pgmap v103: 73 pgs: 73 active+clean; 589 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 372 KiB/s rd, 41 KiB/s wr, 486 op/s 2024-04-05T03:04:37.472 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:37 smithi033 ceph-mon[40008]: osdmap e77: 8 total, 8 up, 8 in 2024-04-05T03:04:38.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:38 smithi033 ceph-mon[29244]: osdmap e78: 8 total, 8 up, 8 in 2024-04-05T03:04:38.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:38 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4125454411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:38.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:38 smithi033 ceph-mon[40008]: osdmap e78: 8 total, 8 up, 8 in 2024-04-05T03:04:38.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:38 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4125454411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:38.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:38 smithi151 ceph-mon[32789]: osdmap e78: 8 total, 8 up, 8 in 2024-04-05T03:04:38.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:38 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4125454411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:38.888 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-05T03:04:38.891 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep 'protected: False' 2024-04-05T03:04:38.891 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd info testimg_import@snap1 2024-04-05T03:04:38.926 INFO:tasks.workunit.client.0.smithi033.stdout: protected: False 2024-04-05T03:04:38.927 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd info testimg_import@snap2 2024-04-05T03:04:38.927 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep 'protected: True' 2024-04-05T03:04:38.965 INFO:tasks.workunit.client.0.smithi033.stdout: protected: True 2024-04-05T03:04:38.965 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/snap_protect 2024-04-05T03:04:38.966 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap unprotect testimg@snap2 2024-04-05T03:04:39.009 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap unprotect testimg_import@snap2 2024-04-05T03:04:39.054 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap purge testimg 2024-04-05T03:04:39.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:39 smithi033 ceph-mon[29244]: pgmap v106: 73 pgs: 73 active+clean; 589 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 41 KiB/s wr, 488 op/s 2024-04-05T03:04:39.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:39 smithi033 ceph-mon[29244]: osdmap e79: 8 total, 8 up, 8 in 2024-04-05T03:04:39.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:39 smithi033 ceph-mon[29244]: osdmap e80: 8 total, 8 up, 8 in 2024-04-05T03:04:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:39 smithi033 ceph-mon[40008]: pgmap v106: 73 pgs: 73 active+clean; 589 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 41 KiB/s wr, 488 op/s 2024-04-05T03:04:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:39 smithi033 ceph-mon[40008]: osdmap e79: 8 total, 8 up, 8 in 2024-04-05T03:04:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:39 smithi033 ceph-mon[40008]: osdmap e80: 8 total, 8 up, 8 in 2024-04-05T03:04:39.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:39 smithi151 ceph-mon[32789]: pgmap v106: 73 pgs: 73 active+clean; 589 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 41 KiB/s wr, 488 op/s 2024-04-05T03:04:39.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:39 smithi151 ceph-mon[32789]: osdmap e79: 8 total, 8 up, 8 in 2024-04-05T03:04:39.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:39 smithi151 ceph-mon[32789]: osdmap e80: 8 total, 8 up, 8 in 2024-04-05T03:04:40.875 INFO:tasks.workunit.client.0.smithi033.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-04-05T03:04:40.884 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd snap purge testimg_import 2024-04-05T03:04:41.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:40 smithi033 ceph-mon[29244]: osdmap e81: 8 total, 8 up, 8 in 2024-04-05T03:04:41.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:40 smithi033 ceph-mon[29244]: pgmap v110: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 18 KiB/s wr, 240 op/s 2024-04-05T03:04:41.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:40 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3370480448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:41.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:40 smithi033 ceph-mon[40008]: osdmap e81: 8 total, 8 up, 8 in 2024-04-05T03:04:41.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:40 smithi033 ceph-mon[40008]: pgmap v110: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 18 KiB/s wr, 240 op/s 2024-04-05T03:04:41.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:40 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3370480448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:41.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:40 smithi151 ceph-mon[32789]: osdmap e81: 8 total, 8 up, 8 in 2024-04-05T03:04:41.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:40 smithi151 ceph-mon[32789]: pgmap v110: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 18 KiB/s wr, 240 op/s 2024-04-05T03:04:41.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:40 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3370480448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:42.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:41 smithi033 ceph-mon[29244]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T03:04:42.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:41 smithi033 ceph-mon[40008]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T03:04:42.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:41 smithi151 ceph-mon[32789]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T03:04:42.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:42 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:42] "GET /metrics HTTP/1.1" 200 239278 "" "Prometheus/2.43.0" 2024-04-05T03:04:42.895 INFO:tasks.workunit.client.0.smithi033.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-04-05T03:04:42.902 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg 2024-04-05T03:04:42.990 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:42.993 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd remove testimg_import 2024-04-05T03:04:43.082 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:43.085 INFO:tasks.workunit.client.0.smithi033.stderr:+ tiered=0 2024-04-05T03:04:43.085 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph osd dump 2024-04-05T03:04:43.085 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep '^pool' 2024-04-05T03:04:43.085 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep ''\''rbd'\''' 2024-04-05T03:04:43.086 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep tier 2024-04-05T03:04:43.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:42 smithi033 ceph-mon[29244]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T03:04:43.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:42 smithi033 ceph-mon[29244]: pgmap v113: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 179 KiB/s rd, 18 KiB/s wr, 241 op/s 2024-04-05T03:04:43.181 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:42 smithi033 ceph-mon[40008]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T03:04:43.181 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:42 smithi033 ceph-mon[40008]: pgmap v113: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 179 KiB/s rd, 18 KiB/s wr, 241 op/s 2024-04-05T03:04:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:42 smithi151 ceph-mon[32789]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T03:04:43.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:42 smithi151 ceph-mon[32789]: pgmap v113: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 179 KiB/s rd, 18 KiB/s wr, 241 op/s 2024-04-05T03:04:43.464 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_81330/sparse1 2024-04-05T03:04:43.468 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records in 2024-04-05T03:04:43.468 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records out 2024-04-05T03:04:43.468 INFO:tasks.workunit.client.0.smithi033.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00326291 s, 321 MB/s 2024-04-05T03:04:43.468 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_81330/sparse2 2024-04-05T03:04:43.472 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records in 2024-04-05T03:04:43.472 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records out 2024-04-05T03:04:43.472 INFO:tasks.workunit.client.0.smithi033.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00317347 s, 330 MB/s 2024-04-05T03:04:43.473 INFO:tasks.workunit.client.0.smithi033.stderr:+ truncate /tmp/rbd_import_export_81330/sparse2 -s 2M 2024-04-05T03:04:43.483 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse1 2024-04-05T03:04:43.523 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 0% complete...failed. 2024-04-05T03:04:43.523 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: delete error: (2) No such file or directory 2024-04-05T03:04:43.526 INFO:tasks.workunit.client.0.smithi033.stderr:+ true 2024-04-05T03:04:43.526 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_81330/sparse1 2024-04-05T03:04:43.587 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-04-05T03:04:43.588 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:43.588 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --order is deprecated, use --object-size 2024-04-05T03:04:43.590 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd ls -l 2024-04-05T03:04:43.590 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep sparse1 2024-04-05T03:04:43.590 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-04-05T03:04:43.625 INFO:tasks.workunit.client.0.smithi033.stdout:sparse1 2 MiB 2 2024-04-05T03:04:43.626 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects sparse1 2024-04-05T03:04:43.626 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse1 2024-04-05T03:04:43.626 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse1 2024-04-05T03:04:43.627 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk '{print $NF;}' 2024-04-05T03:04:43.627 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep block_name_prefix 2024-04-05T03:04:43.663 INFO:tasks.workunit.client.0.smithi033.stderr:++ prefix=rbd_data.6251f2de4ed0 2024-04-05T03:04:43.663 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep rbd_data.6251f2de4ed0 2024-04-05T03:04:43.663 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sed -e 's/rbd_data.6251f2de4ed0\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-05T03:04:43.663 INFO:tasks.workunit.client.0.smithi033.stderr:++++ get_image_data_pool sparse1 2024-04-05T03:04:43.664 INFO:tasks.workunit.client.0.smithi033.stderr:++++ image=sparse1 2024-04-05T03:04:43.664 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sort -u 2024-04-05T03:04:43.664 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ rbd info sparse1 2024-04-05T03:04:43.664 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ grep 'data_pool: ' 2024-04-05T03:04:43.664 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ awk -F: '{ print $NF }' 2024-04-05T03:04:43.699 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool= 2024-04-05T03:04:43.699 INFO:tasks.workunit.client.0.smithi033.stderr:++++ '[' -z ']' 2024-04-05T03:04:43.699 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool=rbd 2024-04-05T03:04:43.699 INFO:tasks.workunit.client.0.smithi033.stderr:++++ echo rbd 2024-04-05T03:04:43.699 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rados ls -p rbd 2024-04-05T03:04:43.730 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects=1 2024-04-05T03:04:43.730 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo 1 2024-04-05T03:04:43.730 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-04-05T03:04:43.730 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export sparse1 /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:43.766 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:43.770 INFO:tasks.workunit.client.0.smithi033.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_81330/sparse1 /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:43.770 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp -l /tmp/rbd_import_export_81330/sparse1 /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:43.771 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse1 --format %b 2024-04-05T03:04:43.772 INFO:tasks.workunit.client.0.smithi033.stderr:+ origsize=2048 2024-04-05T03:04:43.772 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse1.out --format %b 2024-04-05T03:04:43.773 INFO:tasks.workunit.client.0.smithi033.stderr:+ exportsize=2048 2024-04-05T03:04:43.774 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:43.774 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:43.774 INFO:tasks.workunit.client.0.smithi033.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-05T03:04:43.774 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:43.774 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse1 2024-04-05T03:04:43.859 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-04-05T03:04:43.862 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse2 2024-04-05T03:04:43.898 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 0% complete...failed. 2024-04-05T03:04:43.898 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: delete error: (2) No such file or directory 2024-04-05T03:04:43.901 INFO:tasks.workunit.client.0.smithi033.stderr:+ true 2024-04-05T03:04:43.901 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_81330/sparse2 2024-04-05T03:04:43.959 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-04-05T03:04:43.960 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:43.960 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --order is deprecated, use --object-size 2024-04-05T03:04:43.962 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd ls -l 2024-04-05T03:04:43.962 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep sparse2 2024-04-05T03:04:43.962 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-04-05T03:04:43.996 INFO:tasks.workunit.client.0.smithi033.stdout:sparse2 2 MiB 2 2024-04-05T03:04:43.997 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects sparse2 2024-04-05T03:04:43.997 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse2 2024-04-05T03:04:43.997 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse2 2024-04-05T03:04:43.998 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep block_name_prefix 2024-04-05T03:04:43.998 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk '{print $NF;}' 2024-04-05T03:04:44.032 INFO:tasks.workunit.client.0.smithi033.stderr:++ prefix=rbd_data.62845216fac4 2024-04-05T03:04:44.033 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep rbd_data.62845216fac4 2024-04-05T03:04:44.033 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sed -e 's/rbd_data.62845216fac4\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-05T03:04:44.034 INFO:tasks.workunit.client.0.smithi033.stderr:++++ get_image_data_pool sparse2 2024-04-05T03:04:44.034 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sort -u 2024-04-05T03:04:44.034 INFO:tasks.workunit.client.0.smithi033.stderr:++++ image=sparse2 2024-04-05T03:04:44.034 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ rbd info sparse2 2024-04-05T03:04:44.034 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ grep 'data_pool: ' 2024-04-05T03:04:44.034 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ awk -F: '{ print $NF }' 2024-04-05T03:04:44.068 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool= 2024-04-05T03:04:44.068 INFO:tasks.workunit.client.0.smithi033.stderr:++++ '[' -z ']' 2024-04-05T03:04:44.068 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool=rbd 2024-04-05T03:04:44.068 INFO:tasks.workunit.client.0.smithi033.stderr:++++ echo rbd 2024-04-05T03:04:44.068 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rados ls -p rbd 2024-04-05T03:04:44.101 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects=0 2024-04-05T03:04:44.101 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo 0 2024-04-05T03:04:44.101 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-04-05T03:04:44.101 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export sparse2 /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.159 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:44.162 INFO:tasks.workunit.client.0.smithi033.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_81330/sparse2 /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.162 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp -l /tmp/rbd_import_export_81330/sparse2 /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.164 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse2 --format %b 2024-04-05T03:04:44.165 INFO:tasks.workunit.client.0.smithi033.stderr:+ origsize=2048 2024-04-05T03:04:44.165 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse2.out --format %b 2024-04-05T03:04:44.166 INFO:tasks.workunit.client.0.smithi033.stderr:+ exportsize=2048 2024-04-05T03:04:44.166 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:44.166 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:44.166 INFO:tasks.workunit.client.0.smithi033.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-05T03:04:44.167 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.167 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse2 2024-04-05T03:04:44.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[29244]: osdmap e84: 8 total, 8 up, 8 in 2024-04-05T03:04:44.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4208695814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:44.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3383175583' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-05T03:04:44.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3383175583' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-05T03:04:44.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[40008]: osdmap e84: 8 total, 8 up, 8 in 2024-04-05T03:04:44.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4208695814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:44.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3383175583' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-05T03:04:44.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:43 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3383175583' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-05T03:04:44.247 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-04-05T03:04:44.250 INFO:tasks.workunit.client.0.smithi033.stderr:+ truncate /tmp/rbd_import_export_81330/sparse1 -s 10M 2024-04-05T03:04:44.250 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format --order 20 - sparse1 2024-04-05T03:04:44.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:43 smithi151 ceph-mon[32789]: osdmap e84: 8 total, 8 up, 8 in 2024-04-05T03:04:44.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4208695814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:44.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3383175583' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-05T03:04:44.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3383175583' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-05T03:04:44.334 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:44.335 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:44.335 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --order is deprecated, use --object-size 2024-04-05T03:04:44.337 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd ls -l 2024-04-05T03:04:44.337 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep sparse1 2024-04-05T03:04:44.337 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-04-05T03:04:44.371 INFO:tasks.workunit.client.0.smithi033.stdout:sparse1 10 MiB 2 2024-04-05T03:04:44.372 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects sparse1 2024-04-05T03:04:44.372 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse1 2024-04-05T03:04:44.372 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse1 2024-04-05T03:04:44.372 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep block_name_prefix 2024-04-05T03:04:44.373 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk '{print $NF;}' 2024-04-05T03:04:44.409 INFO:tasks.workunit.client.0.smithi033.stderr:++ prefix=rbd_data.3bfaae08df69 2024-04-05T03:04:44.410 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep rbd_data.3bfaae08df69 2024-04-05T03:04:44.411 INFO:tasks.workunit.client.0.smithi033.stderr:++++ get_image_data_pool sparse1 2024-04-05T03:04:44.411 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sed -e 's/rbd_data.3bfaae08df69\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-05T03:04:44.411 INFO:tasks.workunit.client.0.smithi033.stderr:++++ image=sparse1 2024-04-05T03:04:44.411 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sort -u 2024-04-05T03:04:44.411 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ rbd info sparse1 2024-04-05T03:04:44.411 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ grep 'data_pool: ' 2024-04-05T03:04:44.411 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ awk -F: '{ print $NF }' 2024-04-05T03:04:44.446 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool= 2024-04-05T03:04:44.446 INFO:tasks.workunit.client.0.smithi033.stderr:++++ '[' -z ']' 2024-04-05T03:04:44.446 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool=rbd 2024-04-05T03:04:44.446 INFO:tasks.workunit.client.0.smithi033.stderr:++++ echo rbd 2024-04-05T03:04:44.446 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rados ls -p rbd 2024-04-05T03:04:44.479 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects=1 2024-04-05T03:04:44.479 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo 1 2024-04-05T03:04:44.479 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-04-05T03:04:44.479 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export sparse1 /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:44.514 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:44.518 INFO:tasks.workunit.client.0.smithi033.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_81330/sparse1 /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:44.518 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp -l /tmp/rbd_import_export_81330/sparse1 /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:44.524 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse1 --format %b 2024-04-05T03:04:44.525 INFO:tasks.workunit.client.0.smithi033.stderr:+ origsize=2048 2024-04-05T03:04:44.525 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse1.out --format %b 2024-04-05T03:04:44.526 INFO:tasks.workunit.client.0.smithi033.stderr:+ exportsize=2048 2024-04-05T03:04:44.526 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:44.526 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:44.526 INFO:tasks.workunit.client.0.smithi033.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-05T03:04:44.527 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/sparse1.out 2024-04-05T03:04:44.527 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse1 2024-04-05T03:04:44.612 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-04-05T03:04:44.614 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_81330/sparse2 oflag=append conv=notrunc 2024-04-05T03:04:44.621 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records in 2024-04-05T03:04:44.621 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records out 2024-04-05T03:04:44.621 INFO:tasks.workunit.client.0.smithi033.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0058721 s, 357 MB/s 2024-04-05T03:04:44.621 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format --order 20 - sparse2 2024-04-05T03:04:44.694 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:44.695 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:44.695 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --order is deprecated, use --object-size 2024-04-05T03:04:44.697 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd ls -l 2024-04-05T03:04:44.698 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep sparse2 2024-04-05T03:04:44.698 INFO:tasks.workunit.client.0.smithi033.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-04-05T03:04:44.733 INFO:tasks.workunit.client.0.smithi033.stdout:sparse2 4 MiB 2 2024-04-05T03:04:44.734 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects sparse2 2024-04-05T03:04:44.734 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse2 2024-04-05T03:04:44.734 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse2 2024-04-05T03:04:44.734 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep block_name_prefix 2024-04-05T03:04:44.734 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk '{print $NF;}' 2024-04-05T03:04:44.768 INFO:tasks.workunit.client.0.smithi033.stderr:++ prefix=rbd_data.3c278232af51 2024-04-05T03:04:44.769 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep rbd_data.3c278232af51 2024-04-05T03:04:44.769 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sed -e 's/rbd_data.3c278232af51\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-05T03:04:44.769 INFO:tasks.workunit.client.0.smithi033.stderr:++++ get_image_data_pool sparse2 2024-04-05T03:04:44.770 INFO:tasks.workunit.client.0.smithi033.stderr:++++ image=sparse2 2024-04-05T03:04:44.770 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sort -u 2024-04-05T03:04:44.770 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ rbd info sparse2 2024-04-05T03:04:44.770 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ grep 'data_pool: ' 2024-04-05T03:04:44.770 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ awk -F: '{ print $NF }' 2024-04-05T03:04:44.804 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool= 2024-04-05T03:04:44.804 INFO:tasks.workunit.client.0.smithi033.stderr:++++ '[' -z ']' 2024-04-05T03:04:44.804 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool=rbd 2024-04-05T03:04:44.804 INFO:tasks.workunit.client.0.smithi033.stderr:++++ echo rbd 2024-04-05T03:04:44.804 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rados ls -p rbd 2024-04-05T03:04:44.836 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects='0 2024-04-05T03:04:44.836 INFO:tasks.workunit.client.0.smithi033.stderr:2 2024-04-05T03:04:44.836 INFO:tasks.workunit.client.0.smithi033.stderr:3' 2024-04-05T03:04:44.836 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo 0 2 3 2024-04-05T03:04:44.836 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-04-05T03:04:44.836 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export sparse2 /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.871 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-04-05T03:04:44.874 INFO:tasks.workunit.client.0.smithi033.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_81330/sparse2 /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.875 INFO:tasks.workunit.client.0.smithi033.stderr:+ cmp -l /tmp/rbd_import_export_81330/sparse2 /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.877 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse2 --format %b 2024-04-05T03:04:44.878 INFO:tasks.workunit.client.0.smithi033.stderr:+ origsize=6144 2024-04-05T03:04:44.878 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse2.out --format %b 2024-04-05T03:04:44.879 INFO:tasks.workunit.client.0.smithi033.stderr:+ exportsize=6144 2024-04-05T03:04:44.880 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:44.880 INFO:tasks.workunit.client.0.smithi033.stderr:+ difference=0 2024-04-05T03:04:44.880 INFO:tasks.workunit.client.0.smithi033.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-05T03:04:44.880 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/sparse2.out 2024-04-05T03:04:44.880 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse2 2024-04-05T03:04:44.968 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-04-05T03:04:44.971 INFO:tasks.workunit.client.0.smithi033.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-04-05T03:04:44.971 INFO:tasks.workunit.client.0.smithi033.stdout:partially-sparse file imports to partially-sparse image 2024-04-05T03:04:44.971 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_81330/sparse1 sparse 2024-04-05T03:04:45.030 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-04-05T03:04:45.031 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:45.031 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --order is deprecated, use --object-size 2024-04-05T03:04:45.034 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects sparse 2024-04-05T03:04:45.034 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse 2024-04-05T03:04:45.034 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse 2024-04-05T03:04:45.034 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep block_name_prefix 2024-04-05T03:04:45.034 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk '{print $NF;}' 2024-04-05T03:04:45.071 INFO:tasks.workunit.client.0.smithi033.stderr:++ prefix=rbd_data.630ba7225cf8 2024-04-05T03:04:45.072 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep rbd_data.630ba7225cf8 2024-04-05T03:04:45.072 INFO:tasks.workunit.client.0.smithi033.stderr:++++ get_image_data_pool sparse 2024-04-05T03:04:45.072 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sed -e 's/rbd_data.630ba7225cf8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-05T03:04:45.072 INFO:tasks.workunit.client.0.smithi033.stderr:++++ image=sparse 2024-04-05T03:04:45.073 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sort -u 2024-04-05T03:04:45.073 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ rbd info sparse 2024-04-05T03:04:45.073 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ grep 'data_pool: ' 2024-04-05T03:04:45.073 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ awk -F: '{ print $NF }' 2024-04-05T03:04:45.107 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool= 2024-04-05T03:04:45.107 INFO:tasks.workunit.client.0.smithi033.stderr:++++ '[' -z ']' 2024-04-05T03:04:45.107 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool=rbd 2024-04-05T03:04:45.107 INFO:tasks.workunit.client.0.smithi033.stderr:++++ echo rbd 2024-04-05T03:04:45.107 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rados ls -p rbd 2024-04-05T03:04:45.139 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects=1 2024-04-05T03:04:45.139 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo 1 2024-04-05T03:04:45.139 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-04-05T03:04:45.139 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse 2024-04-05T03:04:45.218 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-04-05T03:04:45.221 INFO:tasks.workunit.client.0.smithi033.stdout:zeros import through stdin to sparse image 2024-04-05T03:04:45.221 INFO:tasks.workunit.client.0.smithi033.stderr:+ echo 'zeros import through stdin to sparse image' 2024-04-05T03:04:45.221 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/dev/zero bs=1M count=4 2024-04-05T03:04:45.221 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd import --new-format - sparse 2024-04-05T03:04:45.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[29244]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 165 KiB/s rd, 17 KiB/s wr, 222 op/s 2024-04-05T03:04:45.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:45.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:45.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:45.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:45.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[40008]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 165 KiB/s rd, 17 KiB/s wr, 222 op/s 2024-04-05T03:04:45.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:45.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:45.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:45.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:44 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:45.270 INFO:tasks.workunit.client.0.smithi033.stderr:4+0 records in 2024-04-05T03:04:45.271 INFO:tasks.workunit.client.0.smithi033.stderr:4+0 records out 2024-04-05T03:04:45.271 INFO:tasks.workunit.client.0.smithi033.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0486436 s, 86.2 MB/s 2024-04-05T03:04:45.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:44 smithi151 ceph-mon[32789]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 165 KiB/s rd, 17 KiB/s wr, 222 op/s 2024-04-05T03:04:45.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:45.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:45.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:45.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:44 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:04:45.281 INFO:tasks.workunit.client.0.smithi033.stderr: Importing image: 100% complete...done. 2024-04-05T03:04:45.282 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:45.284 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects sparse 2024-04-05T03:04:45.284 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse 2024-04-05T03:04:45.285 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse 2024-04-05T03:04:45.285 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep block_name_prefix 2024-04-05T03:04:45.285 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk '{print $NF;}' 2024-04-05T03:04:45.321 INFO:tasks.workunit.client.0.smithi033.stderr:++ prefix=rbd_data.3c7b6f59207f 2024-04-05T03:04:45.322 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep rbd_data.3c7b6f59207f 2024-04-05T03:04:45.322 INFO:tasks.workunit.client.0.smithi033.stderr:++++ get_image_data_pool sparse 2024-04-05T03:04:45.322 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sed -e 's/rbd_data.3c7b6f59207f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-05T03:04:45.322 INFO:tasks.workunit.client.0.smithi033.stderr:++++ image=sparse 2024-04-05T03:04:45.322 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sort -u 2024-04-05T03:04:45.323 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ rbd info sparse 2024-04-05T03:04:45.323 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ grep 'data_pool: ' 2024-04-05T03:04:45.323 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ awk -F: '{ print $NF }' 2024-04-05T03:04:45.357 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool= 2024-04-05T03:04:45.357 INFO:tasks.workunit.client.0.smithi033.stderr:++++ '[' -z ']' 2024-04-05T03:04:45.358 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool=rbd 2024-04-05T03:04:45.358 INFO:tasks.workunit.client.0.smithi033.stderr:++++ echo rbd 2024-04-05T03:04:45.358 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rados ls -p rbd 2024-04-05T03:04:45.389 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects= 2024-04-05T03:04:45.389 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo 2024-04-05T03:04:45.389 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-04-05T03:04:45.390 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse 2024-04-05T03:04:45.466 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:45.469 INFO:tasks.workunit.client.0.smithi033.stdout:zeros export to sparse file 2024-04-05T03:04:45.469 INFO:tasks.workunit.client.0.smithi033.stderr:+ echo 'zeros export to sparse file' 2024-04-05T03:04:45.469 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd create --new-format sparse --size 4 2024-04-05T03:04:45.506 INFO:tasks.workunit.client.0.smithi033.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-05T03:04:45.509 INFO:tasks.workunit.client.0.smithi033.stderr:++ rbd info sparse 2024-04-05T03:04:45.509 INFO:tasks.workunit.client.0.smithi033.stderr:++ grep block_name_prefix 2024-04-05T03:04:45.509 INFO:tasks.workunit.client.0.smithi033.stderr:++ awk '{print $NF;}' 2024-04-05T03:04:45.543 INFO:tasks.workunit.client.0.smithi033.stderr:+ prefix=rbd_data.6353fe521281 2024-04-05T03:04:45.543 INFO:tasks.workunit.client.0.smithi033.stderr:+ dd if=/dev/zero bs=4M count=1 2024-04-05T03:04:45.543 INFO:tasks.workunit.client.0.smithi033.stderr:++ get_image_data_pool sparse 2024-04-05T03:04:45.543 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse 2024-04-05T03:04:45.544 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse 2024-04-05T03:04:45.544 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep 'data_pool: ' 2024-04-05T03:04:45.544 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk -F: '{ print $NF }' 2024-04-05T03:04:45.579 INFO:tasks.workunit.client.0.smithi033.stderr:++ data_pool= 2024-04-05T03:04:45.579 INFO:tasks.workunit.client.0.smithi033.stderr:++ '[' -z ']' 2024-04-05T03:04:45.579 INFO:tasks.workunit.client.0.smithi033.stderr:++ data_pool=rbd 2024-04-05T03:04:45.579 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo rbd 2024-04-05T03:04:45.580 INFO:tasks.workunit.client.0.smithi033.stderr:+ rados -p rbd put rbd_data.6353fe521281.000000000000 - 2024-04-05T03:04:45.607 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records in 2024-04-05T03:04:45.608 INFO:tasks.workunit.client.0.smithi033.stderr:1+0 records out 2024-04-05T03:04:45.608 INFO:tasks.workunit.client.0.smithi033.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0640185 s, 65.5 MB/s 2024-04-05T03:04:45.627 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects sparse 2024-04-05T03:04:45.627 INFO:tasks.workunit.client.0.smithi033.stderr:++ image=sparse 2024-04-05T03:04:45.627 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rbd info sparse 2024-04-05T03:04:45.627 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep block_name_prefix 2024-04-05T03:04:45.627 INFO:tasks.workunit.client.0.smithi033.stderr:+++ awk '{print $NF;}' 2024-04-05T03:04:45.661 INFO:tasks.workunit.client.0.smithi033.stderr:++ prefix=rbd_data.6353fe521281 2024-04-05T03:04:45.662 INFO:tasks.workunit.client.0.smithi033.stderr:+++ grep rbd_data.6353fe521281 2024-04-05T03:04:45.662 INFO:tasks.workunit.client.0.smithi033.stderr:++++ get_image_data_pool sparse 2024-04-05T03:04:45.663 INFO:tasks.workunit.client.0.smithi033.stderr:++++ image=sparse 2024-04-05T03:04:45.663 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sed -e 's/rbd_data.6353fe521281\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-04-05T03:04:45.663 INFO:tasks.workunit.client.0.smithi033.stderr:+++ sort -u 2024-04-05T03:04:45.663 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ rbd info sparse 2024-04-05T03:04:45.663 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ grep 'data_pool: ' 2024-04-05T03:04:45.663 INFO:tasks.workunit.client.0.smithi033.stderr:+++++ awk -F: '{ print $NF }' 2024-04-05T03:04:45.697 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool= 2024-04-05T03:04:45.697 INFO:tasks.workunit.client.0.smithi033.stderr:++++ '[' -z ']' 2024-04-05T03:04:45.697 INFO:tasks.workunit.client.0.smithi033.stderr:++++ data_pool=rbd 2024-04-05T03:04:45.697 INFO:tasks.workunit.client.0.smithi033.stderr:++++ echo rbd 2024-04-05T03:04:45.697 INFO:tasks.workunit.client.0.smithi033.stderr:+++ rados ls -p rbd 2024-04-05T03:04:45.728 INFO:tasks.workunit.client.0.smithi033.stderr:++ objects=0 2024-04-05T03:04:45.728 INFO:tasks.workunit.client.0.smithi033.stderr:++ echo 0 2024-04-05T03:04:45.728 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-04-05T03:04:45.728 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/sparse 2024-04-05T03:04:45.729 INFO:tasks.workunit.client.0.smithi033.stderr:rm: cannot remove '/tmp/rbd_import_export_81330/sparse': No such file or directory 2024-04-05T03:04:45.729 INFO:tasks.workunit.client.0.smithi033.stderr:+ true 2024-04-05T03:04:45.729 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd export sparse /tmp/rbd_import_export_81330/sparse 2024-04-05T03:04:45.763 INFO:tasks.workunit.client.0.smithi033.stderr: Exporting image: 100% complete...done. 2024-04-05T03:04:45.766 INFO:tasks.workunit.client.0.smithi033.stderr:++ stat /tmp/rbd_import_export_81330/sparse --format=%b 2024-04-05T03:04:45.767 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 0 = 0 ']' 2024-04-05T03:04:45.767 INFO:tasks.workunit.client.0.smithi033.stderr:+ rbd rm sparse 2024-04-05T03:04:45.846 INFO:tasks.workunit.client.0.smithi033.stderr: Removing image: 100% complete...done. 2024-04-05T03:04:45.849 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm /tmp/rbd_import_export_81330/sparse /tmp/rbd_import_export_81330/sparse1 /tmp/rbd_import_export_81330/sparse2 /tmp/rbd_import_export_81330/sparse3 2024-04-05T03:04:45.851 INFO:tasks.workunit.client.0.smithi033.stderr:rm: cannot remove '/tmp/rbd_import_export_81330/sparse3': No such file or directory 2024-04-05T03:04:45.851 INFO:tasks.workunit.client.0.smithi033.stderr:+ true 2024-04-05T03:04:45.851 INFO:tasks.workunit.client.0.smithi033.stdout:OK 2024-04-05T03:04:45.852 INFO:tasks.workunit.client.0.smithi033.stderr:+ echo OK 2024-04-05T03:04:45.853 INFO:tasks.workunit.client.0.smithi033.stderr:+ rm -rf /tmp/rbd_import_export_81330 2024-04-05T03:04:45.853 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-04-05T03:04:45.853 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-04-05T03:04:45.883 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-04-05T03:04:45.883 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-04-05T03:04:46.138 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[29244]: Upgrade: Target is version 19.0.0-2013-g0028525c (unknown) 2024-04-05T03:04:46.138 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[29244]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca'] 2024-04-05T03:04:46.138 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[29244]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-05T03:04:46.138 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[29244]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca on smithi151 2024-04-05T03:04:46.138 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2712428247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:46.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[40008]: Upgrade: Target is version 19.0.0-2013-g0028525c (unknown) 2024-04-05T03:04:46.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[40008]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca'] 2024-04-05T03:04:46.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[40008]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-05T03:04:46.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[40008]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca on smithi151 2024-04-05T03:04:46.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:45 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2712428247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:46.140 DEBUG:teuthology.parallel:result is None 2024-04-05T03:04:46.140 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-04-05T03:04:46.167 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-04-05T03:04:46.167 DEBUG:teuthology.orchestra.run.smithi033:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-04-05T03:04:46.223 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-04-05T03:04:46.223 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-05T03:04:46.223 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-04-05T03:04:46.223 DEBUG:teuthology.parallel:result is None 2024-04-05T03:04:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:45 smithi151 ceph-mon[32789]: Upgrade: Target is version 19.0.0-2013-g0028525c (unknown) 2024-04-05T03:04:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:45 smithi151 ceph-mon[32789]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca'] 2024-04-05T03:04:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:45 smithi151 ceph-mon[32789]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-05T03:04:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:45 smithi151 ceph-mon[32789]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca on smithi151 2024-04-05T03:04:46.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:45 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2712428247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:47.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:46 smithi033 ceph-mon[29244]: pgmap v116: 73 pgs: 73 active+clean; 1.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 328 op/s 2024-04-05T03:04:47.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:46 smithi033 ceph-mon[40008]: pgmap v116: 73 pgs: 73 active+clean; 1.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 328 op/s 2024-04-05T03:04:47.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:46 smithi151 ceph-mon[32789]: pgmap v116: 73 pgs: 73 active+clean; 1.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 328 op/s 2024-04-05T03:04:47.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:04:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:04:48.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:47 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2442939469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:48.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:47 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2442939469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:48.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:47 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2442939469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:49.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:48 smithi033 ceph-mon[29244]: pgmap v117: 73 pgs: 73 active+clean; 1.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1010 KiB/s wr, 274 op/s 2024-04-05T03:04:49.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:48 smithi033 ceph-mon[29244]: osdmap e85: 8 total, 8 up, 8 in 2024-04-05T03:04:49.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:48 smithi033 ceph-mon[40008]: pgmap v117: 73 pgs: 73 active+clean; 1.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1010 KiB/s wr, 274 op/s 2024-04-05T03:04:49.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:48 smithi033 ceph-mon[40008]: osdmap e85: 8 total, 8 up, 8 in 2024-04-05T03:04:49.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:48 smithi151 ceph-mon[32789]: pgmap v117: 73 pgs: 73 active+clean; 1.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1010 KiB/s wr, 274 op/s 2024-04-05T03:04:49.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:48 smithi151 ceph-mon[32789]: osdmap e85: 8 total, 8 up, 8 in 2024-04-05T03:04:51.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:51 smithi033 ceph-mon[29244]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2024-04-05T03:04:51.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:51 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/587683329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:51.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:51 smithi033 ceph-mon[40008]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2024-04-05T03:04:51.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:51 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/587683329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:51 smithi151 ceph-mon[32789]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2024-04-05T03:04:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:51 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/587683329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:52.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:04:52 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:52] "GET /metrics HTTP/1.1" 200 239985 "" "Prometheus/2.43.0" 2024-04-05T03:04:53.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:53 smithi033 ceph-mon[40008]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 890 KiB/s rd, 1.2 MiB/s wr, 295 op/s 2024-04-05T03:04:53.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:53 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2009583254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:53.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:53 smithi033 ceph-mon[29244]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 890 KiB/s rd, 1.2 MiB/s wr, 295 op/s 2024-04-05T03:04:53.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:53 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2009583254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:53.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:53 smithi151 ceph-mon[32789]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 890 KiB/s rd, 1.2 MiB/s wr, 295 op/s 2024-04-05T03:04:53.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:53 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2009583254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:55.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:55 smithi033 ceph-mon[29244]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-04-05T03:04:55.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:55 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/967520253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:55.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:55 smithi033 ceph-mon[40008]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-04-05T03:04:55.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:55 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/967520253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:55.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:55 smithi151 ceph-mon[32789]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-04-05T03:04:55.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:55 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/967520253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:57.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:57 smithi151 ceph-mon[32789]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 412 KiB/s wr, 73 op/s 2024-04-05T03:04:57.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:04:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:04:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:04:57.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:57 smithi033 ceph-mon[40008]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 412 KiB/s wr, 73 op/s 2024-04-05T03:04:57.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:57 smithi033 ceph-mon[29244]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 412 KiB/s wr, 73 op/s 2024-04-05T03:04:58.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:58 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/483470662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:58.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:58 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/483470662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:58.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:58 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/483470662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:04:59.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:59 smithi033 ceph-mon[29244]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 412 KiB/s wr, 73 op/s 2024-04-05T03:04:59.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:04:59 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:59.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:59 smithi033 ceph-mon[40008]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 412 KiB/s wr, 73 op/s 2024-04-05T03:04:59.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:04:59 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:59.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:59 smithi151 ceph-mon[32789]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 412 KiB/s wr, 73 op/s 2024-04-05T03:04:59.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:04:59 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:00.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:00 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2053490783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:00.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:00 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2053490783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:00.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:00 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2053490783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:01.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:01 smithi033 ceph-mon[29244]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 367 KiB/s wr, 66 op/s 2024-04-05T03:05:01.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:01 smithi033 ceph-mon[40008]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 367 KiB/s wr, 66 op/s 2024-04-05T03:05:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:01 smithi151 ceph-mon[32789]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 367 KiB/s wr, 66 op/s 2024-04-05T03:05:02.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:02 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:02] "GET /metrics HTTP/1.1" 200 240006 "" "Prometheus/2.43.0" 2024-04-05T03:05:03.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:03 smithi033 ceph-mon[29244]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:03.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:03 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2898850987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:03.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:03 smithi033 ceph-mon[40008]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:03.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:03 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2898850987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:03.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:03 smithi151 ceph-mon[32789]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:03.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:03 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2898850987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:05.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:05 smithi033 ceph-mon[29244]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:05.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:05 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/881178131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:05.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:05 smithi033 ceph-mon[40008]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:05.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:05 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/881178131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:05.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:05 smithi151 ceph-mon[32789]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:05.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:05 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/881178131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:07.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:07 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[34668]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:05:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:07 smithi151 ceph-mon[32789]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:07 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2405762681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:07.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:07 smithi033 ceph-mon[29244]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:07.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:07 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2405762681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:07.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:07 smithi033 ceph-mon[40008]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:07.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:07 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2405762681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:09.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:09 smithi033 ceph-mon[40008]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:09.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:09 smithi033 ceph-mon[29244]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:09.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:09 smithi151 ceph-mon[32789]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:10.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:10 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/282905773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:10.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:10 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/282905773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:10.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:10 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/282905773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:11.429 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:11 smithi151 ceph-mon[32789]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:11.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:11 smithi033 ceph-mon[29244]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:11.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:11 smithi033 ceph-mon[40008]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:12.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:12 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2280569317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:12.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:12 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2280569317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:12.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:12 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:12] "GET /metrics HTTP/1.1" 200 240006 "" "Prometheus/2.43.0" 2024-04-05T03:05:12.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:12 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2280569317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:13.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:13 smithi033 ceph-mon[29244]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:13.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:13 smithi033 ceph-mon[40008]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:13.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:13 smithi151 ceph-mon[32789]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:14.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:05:14.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:14 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:14.498 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:14 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:14.498 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:14 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:14.498 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:14 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:05:14.498 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:14 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:05:14.499 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:14 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:15.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[29244]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:15.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[29244]: Upgrade: Updating mgr.x 2024-04-05T03:05:15.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[29244]: Deploying daemon mgr.x on smithi151 2024-04-05T03:05:15.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2029876837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:15.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[40008]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:15.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[40008]: Upgrade: Updating mgr.x 2024-04-05T03:05:15.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[40008]: Deploying daemon mgr.x on smithi151 2024-04-05T03:05:15.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:15 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2029876837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:15 smithi151 ceph-mon[32789]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:15 smithi151 ceph-mon[32789]: Upgrade: Updating mgr.x 2024-04-05T03:05:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:15 smithi151 ceph-mon[32789]: Deploying daemon mgr.x on smithi151 2024-04-05T03:05:15.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:15 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2029876837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:16.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:15 smithi151 systemd[1]: Stopping Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:05:16.295 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 podman[66048]: 2024-04-05 03:05:16.042460113 +0000 UTC m=+0.128438917 container died c743b76f858720fe2d79c98d146d45bc1a6f64f261700760f4f27901221946d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f) 2024-04-05T03:05:16.654 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 podman[66048]: 2024-04-05 03:05:16.387064473 +0000 UTC m=+0.473043285 container cleanup c743b76f858720fe2d79c98d146d45bc1a6f64f261700760f4f27901221946d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-6fda41f, ceph=True, io.buildah.version=1.33.5) 2024-04-05T03:05:16.654 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 bash[66048]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x 2024-04-05T03:05:16.997 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 podman[66062]: 2024-04-05 03:05:16.655556892 +0000 UTC m=+0.617604131 container remove c743b76f858720fe2d79c98d146d45bc1a6f64f261700760f4f27901221946d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2) 2024-04-05T03:05:16.997 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:05:16.997 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-05T03:05:16.997 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 systemd[1]: Stopped Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:05:16.997 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service: Consumed 17.478s CPU time. 2024-04-05T03:05:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:17 smithi151 ceph-mon[32789]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:17 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3951857549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:17.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:16 smithi151 systemd[1]: Starting Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:05:17.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:17 smithi151 podman[66194]: 2024-04-05 03:05:17.113521298 +0000 UTC m=+0.017746832 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:05:17.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:17 smithi151 podman[66194]: 2024-04-05 03:05:17.240341722 +0000 UTC m=+0.144567302 container create 80ca6768c17e2b56da0bfb677e527c97f2cfd8cbdf86babe88bde936c79bff9d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:05:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:17 smithi033 ceph-mon[29244]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:17 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3951857549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:17.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:17 smithi033 ceph-mon[40008]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:17.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:17 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3951857549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:17.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:17 smithi151 podman[66194]: 2024-04-05 03:05:17.443120766 +0000 UTC m=+0.347346302 container init 80ca6768c17e2b56da0bfb677e527c97f2cfd8cbdf86babe88bde936c79bff9d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-05T03:05:17.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:17 smithi151 podman[66194]: 2024-04-05 03:05:17.445653446 +0000 UTC m=+0.349878981 container start 80ca6768c17e2b56da0bfb677e527c97f2cfd8cbdf86babe88bde936c79bff9d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-05T03:05:17.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:17 smithi151 ceph-mgr[66213]: -- 172.21.15.151:0/3664464336 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d38ada25a0 con 0x55d38ad76800 2024-04-05T03:05:17.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:17 smithi151 bash[66194]: 80ca6768c17e2b56da0bfb677e527c97f2cfd8cbdf86babe88bde936c79bff9d 2024-04-05T03:05:17.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:17 smithi151 systemd[1]: Started Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:05:18.121 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:18.018+0000 7f3df89a9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:05:18.392 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:18.119+0000 7f3df89a9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:05:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-mon[32789]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:18.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:18.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[29244]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:18.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:18.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:18.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:18.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:18 smithi033 ceph-mon[40008]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:19.044 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:18.796+0000 7f3df89a9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:05:19.044 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:18.895+0000 7f3df89a9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:05:19.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:19.044+0000 7f3df89a9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:05:19.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:19.150+0000 7f3df89a9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:05:19.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:19.240+0000 7f3df89a9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:05:19.850 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4050603365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:19.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:19.535+0000 7f3df89a9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:05:19.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:19.693+0000 7f3df89a9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:05:19.850 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:19.785+0000 7f3df89a9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:05:19.908 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:19 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4050603365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:19.908 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:19 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4050603365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:20.104 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:19 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:19.887+0000 7f3df89a9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:05:20.104 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:20 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:20.072+0000 7f3df89a9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:05:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:20 smithi151 ceph-mon[32789]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:20 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:20 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:20.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:20 smithi033 ceph-mon[29244]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:20.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:20 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:20.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:20 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:20.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:20 smithi033 ceph-mon[40008]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:20.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:20 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:20.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:20 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:21.464 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:21.120+0000 7f3df89a9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:05:21.465 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:21.258+0000 7f3df89a9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:05:21.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:21 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1953205775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:21.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:21.602+0000 7f3df89a9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:05:21.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:21.701+0000 7f3df89a9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:05:21.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:21 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1953205775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:21.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:21 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1953205775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:22.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:21.916+0000 7f3df89a9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:05:22.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:22.120+0000 7f3df89a9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:05:22.711 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:22.428+0000 7f3df89a9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:05:22.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:22] "GET /metrics HTTP/1.1" 200 239972 "" "Prometheus/2.43.0" 2024-04-05T03:05:22.963 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:22.711+0000 7f3df89a9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:05:22.963 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:22.963+0000 7f3df89a9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:05:22.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:22.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:22.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-mon[32789]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:22.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:22.963 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:22 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[40008]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[29244]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:23.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:22 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:24.090 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:23 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:23.804+0000 7f3df89a9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:05:24.090 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:23 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1585434979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:24.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:23 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1585434979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:24.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:23 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1585434979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:24.743 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:24.408+0000 7f3df89a9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:05:24.743 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:24.512+0000 7f3df89a9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:05:25.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-mon[32789]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:25.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-mon[32789]: Standby manager daemon x restarted 2024-04-05T03:05:25.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-mon[32789]: Standby manager daemon x started 2024-04-05T03:05:25.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:05:25.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:05:25.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:05:25.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:05:25.026 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:24.743+0000 7f3df89a9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:05:25.026 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:24] ENGINE Bus STARTING 2024-04-05T03:05:25.026 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: CherryPy Checker: 2024-04-05T03:05:25.026 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: The Application mounted at '' has an empty config. 2024-04-05T03:05:25.026 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:25.026 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:24] ENGINE Serving on http://:::9283 2024-04-05T03:05:25.026 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:24 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:24] ENGINE Bus STARTED 2024-04-05T03:05:25.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[29244]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[29244]: Standby manager daemon x restarted 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[29244]: Standby manager daemon x started 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[40008]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[40008]: Standby manager daemon x restarted 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[40008]: Standby manager daemon x started 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:05:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:05:25.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:05:25.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:24 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:05:26.002 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-mon[32789]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:25] ENGINE Bus STOPPING 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:25 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:25] ENGINE Bus STOPPED 2024-04-05T03:05:26.003 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:26] ENGINE Bus STARTING 2024-04-05T03:05:26.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[40008]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-05T03:05:26.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:26.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:05:26.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.127 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[29244]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setuser ceph since I am not root 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ignoring --setgroup ceph since I am not root 2024-04-05T03:05:26.128 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:25 smithi033 ceph-mgr[29477]: -- 172.21.15.33:0/2618349595 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cfc28049c0 con 0x55cfc27da800 2024-04-05T03:05:26.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d58641700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d58641700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d58641700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d58641700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d58641700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d5663d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d5663d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d5663d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d5663d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:05:26.055+0000 7f3d5663d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:05:26.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:26] ENGINE Serving on http://:::9283 2024-04-05T03:05:26.279 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:05:26] ENGINE Bus STARTED 2024-04-05T03:05:26.876 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:26.484+0000 7f0a6f4101c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:05:26.876 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:26.578+0000 7f0a6f4101c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:05:27.228 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:27.228 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: osdmap e86: 8 total, 8 up, 8 in 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: mgrmap e22: x(active, starting, since 0.00668224s) 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:05:27.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: Manager daemon x is now available 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:05:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: osdmap e86: 8 total, 8 up, 8 in 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: mgrmap e22: x(active, starting, since 0.00668224s) 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:05:27.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: Manager daemon x is now available 2024-04-05T03:05:27.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2134633714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: Deploying cephadm binary to smithi033 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[40008]: Deploying cephadm binary to smithi151 2024-04-05T03:05:27.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T03:05:27.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T03:05:27.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2134633714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:27.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: Deploying cephadm binary to smithi033 2024-04-05T03:05:27.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:26 smithi033 ceph-mon[29244]: Deploying cephadm binary to smithi151 2024-04-05T03:05:27.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:27 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:27] "GET /metrics HTTP/1.1" 200 31006 "" "Prometheus/2.43.0" 2024-04-05T03:05:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T03:05:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: osdmap e86: 8 total, 8 up, 8 in 2024-04-05T03:05:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.14448 172.21.15.33:0/3082987484' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-05T03:05:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: mgrmap e22: x(active, starting, since 0.00668224s) 2024-04-05T03:05:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:05:27.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:05:27.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:05:27.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:05:27.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:05:27.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:05:27.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:05:27.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:05:27.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:05:27.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:05:27.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: Manager daemon x is now available 2024-04-05T03:05:27.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:27.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:27.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:05:27.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:05:27.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T03:05:27.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T03:05:27.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2134633714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:27.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: Deploying cephadm binary to smithi033 2024-04-05T03:05:27.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:26 smithi151 ceph-mon[32789]: Deploying cephadm binary to smithi151 2024-04-05T03:05:27.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:27.227+0000 7f0a6f4101c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:05:27.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:27.320+0000 7f0a6f4101c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:05:27.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:27.407+0000 7f0a6f4101c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:05:27.817 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:27.560+0000 7f0a6f4101c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:05:27.817 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:27.642+0000 7f0a6f4101c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[40008]: mgrmap e23: x(active, since 1.00823s) 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[40008]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[29244]: mgrmap e23: x(active, since 1.00823s) 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[29244]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:28.093 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:28.094 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:27.817+0000 7f0a6f4101c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:05:28.094 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:28.004+0000 7f0a6f4101c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:05:28.094 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:28.092+0000 7f0a6f4101c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:05:28.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:27 smithi151 ceph-mon[32789]: mgrmap e23: x(active, since 1.00823s) 2024-04-05T03:05:28.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:27 smithi151 ceph-mon[32789]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:28.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:27 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:28.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:27 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:28.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:28.188+0000 7f0a6f4101c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:05:28.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:28.279+0000 7f0a6f4101c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:05:28.936 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:05:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"172.21.15.33:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:05:28.937 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:05:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephPGImbalance index=15 msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on (job) group_left () avg by (job) (ceph_osd_numpg\n > 0)) / on (job) group_left () avg by (job) (ceph_osd_numpg > 0)) * on (ceph_daemon)\n group_left (hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"172.21.15.33:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:05:29.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[29244]: [05/Apr/2024:03:05:27] ENGINE Bus STARTING 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[29244]: [05/Apr/2024:03:05:27] ENGINE Serving on https://172.21.15.151:7150 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[29244]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[29244]: [05/Apr/2024:03:05:27] ENGINE Serving on http://172.21.15.151:8765 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[29244]: [05/Apr/2024:03:05:27] ENGINE Bus STARTED 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2492273059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[40008]: [05/Apr/2024:03:05:27] ENGINE Bus STARTING 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[40008]: [05/Apr/2024:03:05:27] ENGINE Serving on https://172.21.15.151:7150 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[40008]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[40008]: [05/Apr/2024:03:05:27] ENGINE Serving on http://172.21.15.151:8765 2024-04-05T03:05:29.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[40008]: [05/Apr/2024:03:05:27] ENGINE Bus STARTED 2024-04-05T03:05:29.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:28 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2492273059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-mon[32789]: [05/Apr/2024:03:05:27] ENGINE Bus STARTING 2024-04-05T03:05:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-mon[32789]: [05/Apr/2024:03:05:27] ENGINE Serving on https://172.21.15.151:7150 2024-04-05T03:05:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-mon[32789]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-mon[32789]: [05/Apr/2024:03:05:27] ENGINE Serving on http://172.21.15.151:8765 2024-04-05T03:05:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-mon[32789]: [05/Apr/2024:03:05:27] ENGINE Bus STARTED 2024-04-05T03:05:29.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:28 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2492273059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:29.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:29.392+0000 7f0a6f4101c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:05:29.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:29.538+0000 7f0a6f4101c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:05:30.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:29 smithi033 ceph-mon[29244]: mgrmap e24: x(active, since 3s) 2024-04-05T03:05:30.153 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:29 smithi033 ceph-mon[40008]: mgrmap e24: x(active, since 3s) 2024-04-05T03:05:30.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:29.737+0000 7f0a6f4101c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:05:30.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:29.972+0000 7f0a6f4101c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:05:30.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:30.152+0000 7f0a6f4101c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:05:30.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:29 smithi151 ceph-mon[32789]: mgrmap e24: x(active, since 3s) 2024-04-05T03:05:30.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:30.339+0000 7f0a6f4101c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:05:30.942 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:30.609+0000 7f0a6f4101c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:05:30.942 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:30.699+0000 7f0a6f4101c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:05:31.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:30 smithi033 ceph-mon[29244]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:31.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:30 smithi033 ceph-mon[40008]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:31 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:31.106+0000 7f0a6f4101c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:05:31.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:30 smithi151 ceph-mon[32789]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:32.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:31 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/959071197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:32.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:31 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/959071197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:32.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:31 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:31.958+0000 7f0a6f4101c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:05:32.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:31 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/959071197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:32.635 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:32.274+0000 7f0a6f4101c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:05:32.635 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:32.635+0000 7f0a6f4101c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:05:32.947 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:32.844+0000 7f0a6f4101c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:05:32.947 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:05:32] ENGINE Bus STARTING 2024-04-05T03:05:33.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:32 smithi151 ceph-mon[32789]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:33.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:32 smithi151 ceph-mon[32789]: Standby manager daemon y started 2024-04-05T03:05:33.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:32 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T03:05:33.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:32 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:05:33.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:32 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T03:05:33.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:32 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:05:33.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[40008]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:33.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[40008]: Standby manager daemon y started 2024-04-05T03:05:33.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T03:05:33.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:05:33.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T03:05:33.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:05:33.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[29244]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:33.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[29244]: Standby manager daemon y started 2024-04-05T03:05:33.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T03:05:33.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:05:33.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T03:05:33.233 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2324422980' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:05:33.233 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: CherryPy Checker: 2024-04-05T03:05:33.233 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: The Application mounted at '' has an empty config. 2024-04-05T03:05:33.233 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: 2024-04-05T03:05:33.233 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:05:32] ENGINE Serving on http://:::9283 2024-04-05T03:05:33.233 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: [05/Apr/2024:03:05:32] ENGINE Bus STARTED 2024-04-05T03:05:34.222 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:33 smithi151 ceph-mon[32789]: mgrmap e25: x(active, since 7s), standbys: y 2024-04-05T03:05:34.223 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:33 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:05:34.223 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:33 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/402780675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:34.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:33 smithi033 ceph-mon[29244]: mgrmap e25: x(active, since 7s), standbys: y 2024-04-05T03:05:34.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:33 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:05:34.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:33 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/402780675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:34.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:33 smithi033 ceph-mon[40008]: mgrmap e25: x(active, since 7s), standbys: y 2024-04-05T03:05:34.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:33 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:05:34.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:33 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/402780675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:35.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:35 smithi033 ceph-mon[40008]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:05:35.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:35 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:35.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:35 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:35.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:35 smithi033 ceph-mon[29244]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:05:35.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:35 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:35.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:35 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:35.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:35 smithi151 ceph-mon[32789]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:05:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:35 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:35 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi033 to 2481M 2024-04-05T03:05:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1313363099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi033 to 2481M 2024-04-05T03:05:36.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:36 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1313363099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:36.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi033 to 2481M 2024-04-05T03:05:36.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:36.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:36 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1313363099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:37.241 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:37] "GET /metrics HTTP/1.1" 200 31006 "" "Prometheus/2.43.0" 2024-04-05T03:05:37.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:37 smithi151 ceph-mon[32789]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:05:37.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:37 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:37.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:37 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:37 smithi033 ceph-mon[40008]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:05:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:37 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:37 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:37.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:37 smithi033 ceph-mon[29244]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:05:37.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:37 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:37.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:37 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:38.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:38 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2565453917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:38.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:38 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2565453917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:38.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:38 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2565453917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:39.575 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:39 smithi033 ceph-mon[29244]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-05T03:05:39.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:39 smithi033 ceph-mon[40008]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-05T03:05:39.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:39 smithi151 ceph-mon[32789]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-05T03:05:41.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:41 smithi151 ceph-mon[32789]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:41.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:41 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/542121588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:41.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:41 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:41 smithi033 ceph-mon[29244]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:41 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/542121588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:41 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:41.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:41 smithi033 ceph-mon[40008]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:41.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:41 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/542121588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:41.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:41 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:42.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:42 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:05:43.520 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:43 smithi151 ceph-mon[32789]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:43.520 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:43 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3717457476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:43.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:43 smithi033 ceph-mon[40008]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:43.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:43 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3717457476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:43.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:43 smithi033 ceph-mon[29244]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:43.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:43 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3717457476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:45.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:45 smithi033 ceph-mon[40008]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:45.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:45 smithi033 ceph-mon[29244]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:45 smithi151 ceph-mon[32789]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:05:46.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:46 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2948214597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:46.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:46 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2948214597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:46 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2948214597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:47.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:47] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2024-04-05T03:05:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:47 smithi033 ceph-mon[29244]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:47 smithi033 ceph-mon[40008]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:47 smithi151 ceph-mon[32789]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:48.647 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:48 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2309139450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:48 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2309139450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:48 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2309139450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:49.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:49 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T03:05:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:05:49.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:49 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: Adjusting osd_memory_target on smithi151 to 2480M 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.595 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: Adjusting osd_memory_target on smithi151 to 2480M 2024-04-05T03:05:50.596 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.598 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.598 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.598 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:50 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: Adjusting osd_memory_target on smithi151 to 2480M 2024-04-05T03:05:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:05:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:05:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:05:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:05:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:50.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:50 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:51.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:05:51.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:05:51.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[40008]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:51.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[40008]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T03:05:51.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[40008]: Reconfiguring daemon alertmanager.a on smithi033 2024-04-05T03:05:51.690 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/148022854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:51.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:05:51.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:05:51.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[29244]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:51.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[29244]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T03:05:51.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[29244]: Reconfiguring daemon alertmanager.a on smithi033 2024-04-05T03:05:51.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:51 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/148022854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:51 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:05:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:51 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:05:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:51 smithi151 ceph-mon[32789]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:51 smithi151 ceph-mon[32789]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T03:05:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:51 smithi151 ceph-mon[32789]: Reconfiguring daemon alertmanager.a on smithi033 2024-04-05T03:05:51.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:51 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/148022854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:52.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:05:52 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[29473]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:05:53.343 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:53 smithi033 systemd[1]: Stopping Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:05:53.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:53 smithi033 ceph-mon[29244]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:53.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:53 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/583237307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:53 smithi033 ceph-mon[40008]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:53 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/583237307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:53.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:53 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[74031]: ts=2024-04-05T03:05:53.441Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:05:53.731 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:53 smithi033 podman[87399]: 2024-04-05 03:05:53.482951889 +0000 UTC m=+0.152001909 container died fc398a28fe815d7bc08205e8aabf1a83913d654cb1c637bef3116eabca82f4c4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:53.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:53 smithi151 ceph-mon[32789]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:53.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:53 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/583237307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:54.117 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:53 smithi033 podman[87399]: 2024-04-05 03:05:53.875727145 +0000 UTC m=+0.544777151 container cleanup fc398a28fe815d7bc08205e8aabf1a83913d654cb1c637bef3116eabca82f4c4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:54.118 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:53 smithi033 bash[87399]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-05T03:05:54.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 podman[87414]: 2024-04-05 03:05:54.11898573 +0000 UTC m=+0.634726592 container remove fc398a28fe815d7bc08205e8aabf1a83913d654cb1c637bef3116eabca82f4c4 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:54.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 podman[87414]: 2024-04-05 03:05:54.202857651 +0000 UTC m=+0.718598469 volume remove 40e20bd7e14ef91247318ed0ac0475fafef9983bd64aaaa6c82b283f03606deb 2024-04-05T03:05:54.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-05T03:05:54.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 systemd[1]: Stopped Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:05:54.481 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@alertmanager.a.service: Consumed 1.031s CPU time. 2024-04-05T03:05:54.481 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 systemd[1]: Starting Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:05:54.872 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 podman[87508]: 2024-04-05 03:05:54.474860548 +0000 UTC m=+0.019075845 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-05T03:05:54.873 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 podman[87508]: 2024-04-05 03:05:54.579110183 +0000 UTC m=+0.123325458 volume create 58cd73f5c8d771e0d4f95555e9fb2a7b64f5aff2194792b055999d26d3ad9488 2024-04-05T03:05:54.873 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:54 smithi033 podman[87508]: 2024-04-05 03:05:54.679482168 +0000 UTC m=+0.223697485 container create a9cf7c76c5199cfdd4bef93e305487b070a336fd4caaa70829f6fe9a947b387b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:55.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:54 smithi033 ceph-mon[40008]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:55.126 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:54 smithi033 ceph-mon[29244]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 podman[87508]: 2024-04-05 03:05:55.024397839 +0000 UTC m=+0.568613117 container init a9cf7c76c5199cfdd4bef93e305487b070a336fd4caaa70829f6fe9a947b387b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 podman[87508]: 2024-04-05 03:05:55.027043035 +0000 UTC m=+0.571258323 container start a9cf7c76c5199cfdd4bef93e305487b070a336fd4caaa70829f6fe9a947b387b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.039Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.039Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.040Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.33 port=9094 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.041Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.065Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T03:05:55.126 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.065Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T03:05:55.127 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.068Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-05T03:05:55.127 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:55.068Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-05T03:05:55.127 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 bash[87508]: a9cf7c76c5199cfdd4bef93e305487b070a336fd4caaa70829f6fe9a947b387b 2024-04-05T03:05:55.127 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:55 smithi033 systemd[1]: Started Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:05:55.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:54 smithi151 ceph-mon[32789]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:55.929 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:55 smithi033 systemd[1]: Stopping Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:05:56.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:56.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:56.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[29244]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-05T03:05:56.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[29244]: Reconfiguring daemon node-exporter.a on smithi033 2024-04-05T03:05:56.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3593667159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:56.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:56.229 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:56.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:56.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[40008]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-05T03:05:56.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[40008]: Reconfiguring daemon node-exporter.a on smithi033 2024-04-05T03:05:56.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3593667159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:56.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:56 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:56.230 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:55 smithi033 podman[87690]: 2024-04-05 03:05:55.930426526 +0000 UTC m=+0.112582713 container died 94bec9c472c2c12fca24d82386a4dea7f425bde989ef1efad6166d2f349feaa7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:56.480 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 podman[87690]: 2024-04-05 03:05:56.336579107 +0000 UTC m=+0.518735291 container cleanup 94bec9c472c2c12fca24d82386a4dea7f425bde989ef1efad6166d2f349feaa7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:56.480 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 bash[87690]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a 2024-04-05T03:05:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:56 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:56 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:56 smithi151 ceph-mon[32789]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-05T03:05:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:56 smithi151 ceph-mon[32789]: Reconfiguring daemon node-exporter.a on smithi033 2024-04-05T03:05:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:56 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3593667159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:56 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:56.980 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 podman[87705]: 2024-04-05 03:05:56.58829674 +0000 UTC m=+0.663314317 container remove 94bec9c472c2c12fca24d82386a4dea7f425bde989ef1efad6166d2f349feaa7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:56.980 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:05:56.981 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-05T03:05:56.981 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 systemd[1]: Stopped Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:05:56.981 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.a.service: Consumed 2.071s CPU time. 2024-04-05T03:05:56.981 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:56 smithi033 systemd[1]: Starting Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:05:57.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:57 smithi151 ceph-mon[32789]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:57.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:05:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:05:57] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.43.0" 2024-04-05T03:05:57.324 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:05:57.042Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000375594s 2024-04-05T03:05:57.324 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-mon[29244]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:57.324 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-mon[40008]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 podman[87802]: 2024-04-05 03:05:56.920064062 +0000 UTC m=+0.017361598 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 podman[87802]: 2024-04-05 03:05:57.023014473 +0000 UTC m=+0.120312036 container create 6da1bdc228ef38d7f121f391c9f2b69ef55931ecc6207b5e821b57a92c19fbe8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 podman[87802]: 2024-04-05 03:05:57.233649442 +0000 UTC m=+0.330946972 container init 6da1bdc228ef38d7f121f391c9f2b69ef55931ecc6207b5e821b57a92c19fbe8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 podman[87802]: 2024-04-05 03:05:57.236567486 +0000 UTC m=+0.333865019 container start 6da1bdc228ef38d7f121f391c9f2b69ef55931ecc6207b5e821b57a92c19fbe8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.240Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.240Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.240Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T03:05:57.325 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.240Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T03:05:57.326 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T03:05:57.327 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T03:05:57.328 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.241Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.242Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.242Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.242Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.242Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.242Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a[87816]: ts=2024-04-05T03:05:57.242Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T03:05:57.329 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 bash[87802]: 6da1bdc228ef38d7f121f391c9f2b69ef55931ecc6207b5e821b57a92c19fbe8 2024-04-05T03:05:57.730 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:05:57 smithi033 systemd[1]: Started Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:05:58.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:58 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:58.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:58 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:58.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:58 smithi151 ceph-mon[32789]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:05:58.458 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:58 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2682167806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:58.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:58.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:58.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[29244]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:05:58.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2682167806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:58.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:58.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:05:58.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[40008]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:05:58.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:58 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2682167806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:05:59.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:59 smithi151 ceph-mon[32789]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:05:59.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:05:59 smithi151 ceph-mon[32789]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:59.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:59 smithi033 ceph-mon[29244]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:05:59.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:05:59 smithi033 ceph-mon[29244]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:05:59.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:59 smithi033 ceph-mon[40008]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:05:59.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:05:59 smithi033 ceph-mon[40008]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:00.672 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:00 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1175419506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:00.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:00 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1175419506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:00.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:00 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1175419506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:01.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:00 smithi151 systemd[1]: Stopping Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:01.380 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.085Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:06:01.380 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T03:06:01.380 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T03:06:01.380 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T03:06:01.380 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T03:06:01.380 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T03:06:01.381 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T03:06:01.381 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T03:06:01.381 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T03:06:01.381 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T03:06:01.381 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T03:06:01.381 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[63396]: ts=2024-04-05T03:06:01.086Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T03:06:01.381 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 podman[74203]: 2024-04-05 03:06:01.096154787 +0000 UTC m=+0.118069916 container died c19cfcd8e6480c422454be1146b535fd0dab98f583ac7c32a47a20fb13dc9aa8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:01.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:01 smithi033 ceph-mon[29244]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:01.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:01 smithi033 ceph-mon[40008]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:01.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:01 smithi151 ceph-mon[32789]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:01.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 podman[74203]: 2024-04-05 03:06:01.570078523 +0000 UTC m=+0.591993701 container cleanup c19cfcd8e6480c422454be1146b535fd0dab98f583ac7c32a47a20fb13dc9aa8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:01.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 bash[74203]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T03:06:02.188 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 podman[74218]: 2024-04-05 03:06:01.84697576 +0000 UTC m=+0.754038571 container remove c19cfcd8e6480c422454be1146b535fd0dab98f583ac7c32a47a20fb13dc9aa8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:02.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-05T03:06:02.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 systemd[1]: Stopped Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:02.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service: Consumed 1.572s CPU time. 2024-04-05T03:06:02.189 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:01 smithi151 systemd[1]: Starting Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:02.451 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 podman[74312]: 2024-04-05 03:06:02.093627658 +0000 UTC m=+0.018753671 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T03:06:02.452 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 podman[74312]: 2024-04-05 03:06:02.232023502 +0000 UTC m=+0.157149515 container create 2f71aa61c3374a18ea5c8c8ff9f63fbec73d3d48e8ab14f6e19e7306d0cb5c5f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:02.452 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 podman[74312]: 2024-04-05 03:06:02.426316958 +0000 UTC m=+0.351442974 container init 2f71aa61c3374a18ea5c8c8ff9f63fbec73d3d48e8ab14f6e19e7306d0cb5c5f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:02.452 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 podman[74312]: 2024-04-05 03:06:02.428887123 +0000 UTC m=+0.354013139 container start 2f71aa61c3374a18ea5c8c8ff9f63fbec73d3d48e8ab14f6e19e7306d0cb5c5f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:02.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.452Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.452Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.452Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi151 (none))" 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.452Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.452Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.454Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.455Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.457Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.457Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.458Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.458Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.201µs 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.458Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.459Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-05T03:06:02.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.472Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.472Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.472Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.265µs wal_replay_duration=14.343535ms wbl_replay_duration=97ns total_replay_duration=14.399131ms 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.475Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.475Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.475Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.493Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.668613ms db_storage=886ns remote_storage=1.098µs web_handler=450ns query_engine=16.063µs scrape=574.724µs scrape_sd=90.54µs notify=27.199µs notify_sd=8.456µs rules=17.661427ms tracing=5.314µs 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.494Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:02.494Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 bash[74312]: 2f71aa61c3374a18ea5c8c8ff9f63fbec73d3d48e8ab14f6e19e7306d0cb5c5f 2024-04-05T03:06:02.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:02 smithi151 systemd[1]: Started Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:03.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:03 smithi151 ceph-mon[32789]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:03.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:03 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2502547990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:03.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:03 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:03.668 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:03 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:03.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[40008]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:03.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2502547990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:03.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:03.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:03.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[29244]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:03.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2502547990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:03.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:03.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:03 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:04.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:04 smithi033 ceph-mon[40008]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T03:06:04.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:04 smithi033 ceph-mon[40008]: Reconfiguring daemon grafana.a on smithi151 2024-04-05T03:06:04.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:04 smithi033 ceph-mon[29244]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T03:06:04.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:04 smithi033 ceph-mon[29244]: Reconfiguring daemon grafana.a on smithi151 2024-04-05T03:06:04.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:04 smithi151 ceph-mon[32789]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T03:06:04.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:04 smithi151 ceph-mon[32789]: Reconfiguring daemon grafana.a on smithi151 2024-04-05T03:06:05.388 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:06:05 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:06:05.044Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003294193s 2024-04-05T03:06:05.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:05 smithi033 ceph-mon[29244]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:05.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:05 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/755524628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:05.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:05 smithi033 ceph-mon[40008]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:05.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:05 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/755524628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:05.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:05 smithi151 ceph-mon[32789]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:05.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:05 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/755524628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:07.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:07 smithi033 ceph-mon[29244]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:07.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:07 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3806403052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:07.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:07 smithi033 ceph-mon[40008]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:07.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:07 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3806403052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:07.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:07 smithi151 ceph-mon[32789]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:07.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:07 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3806403052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:09.670 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:09 smithi151 ceph-mon[32789]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:09.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:09 smithi033 ceph-mon[29244]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:09.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:09 smithi033 ceph-mon[40008]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:10.673 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:10 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3662192454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:10.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:10 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3662192454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:10.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:10 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3662192454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:11.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:06:11] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2024-04-05T03:06:11.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:11 smithi151 ceph-mon[32789]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:11.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:11 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:11.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:11 smithi033 ceph-mon[29244]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:11.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:11 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:11.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:11 smithi033 ceph-mon[40008]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:11.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:11 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:12.493 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:12 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3789257816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:12.493 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:11 smithi151 systemd[1]: Stopping Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:12.493 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:12 smithi151 podman[74651]: 2024-04-05 03:06:12.211314132 +0000 UTC m=+0.108690626 container died a5ab46c212de9f98990643abd945eebb5928e359a2dc0d1a19ba9ca05d63628b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, architecture=x86_64, vcs-type=git, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, version=8.5, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T03:06:12.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:12 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3789257816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:12.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:12 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3789257816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:12.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:12 smithi151 podman[74651]: 2024-04-05 03:06:12.567022087 +0000 UTC m=+0.464398596 container cleanup a5ab46c212de9f98990643abd945eebb5928e359a2dc0d1a19ba9ca05d63628b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.openshift.tags=base rhel8, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner ) 2024-04-05T03:06:12.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:12 smithi151 bash[74651]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a 2024-04-05T03:06:13.251 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:12 smithi151 podman[74665]: 2024-04-05 03:06:12.877052702 +0000 UTC m=+0.664854071 container remove a5ab46c212de9f98990643abd945eebb5928e359a2dc0d1a19ba9ca05d63628b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, release=239.1651231664, architecture=x86_64, io.openshift.expose-services=, maintainer=Paul Cuzner , distribution-scope=public, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-05T03:06:13.251 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:12 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:06:13.251 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:13 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:06:13.251 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:13 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:13.251 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:13 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 5.100s CPU time. 2024-04-05T03:06:13.251 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:13 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:13 smithi151 ceph-mon[32789]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:13.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:13 smithi151 podman[74764]: 2024-04-05 03:06:13.154642524 +0000 UTC m=+0.017332714 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:06:13.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:13 smithi033 ceph-mon[29244]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:13.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:13 smithi033 ceph-mon[40008]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:14.590 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:14 smithi151 podman[74764]: 2024-04-05 03:06:14.280490125 +0000 UTC m=+1.143180357 container create 30170eec85dcf2157dd9cc27a77c7537fb684179c81852b08fc386c92400ea0c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , architecture=x86_64, io.openshift.tags=base rhel8, io.openshift.expose-services=, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897) 2024-04-05T03:06:14.590 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:14 smithi151 podman[74764]: 2024-04-05 03:06:14.499318941 +0000 UTC m=+1.362009131 container init 30170eec85dcf2157dd9cc27a77c7537fb684179c81852b08fc386c92400ea0c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , name=ubi8, architecture=x86_64, release=239.1651231664, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, version=8.5) 2024-04-05T03:06:14.590 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:14 smithi151 podman[74764]: 2024-04-05 03:06:14.502399999 +0000 UTC m=+1.365090199 container start 30170eec85dcf2157dd9cc27a77c7537fb684179c81852b08fc386c92400ea0c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, architecture=x86_64, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, io.openshift.tags=base rhel8, version=8.5, vendor=Red Hat, Inc., description=Ceph Grafana Container, release=239.1651231664) 2024-04-05T03:06:15.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:14 smithi151 bash[74764]: 30170eec85dcf2157dd9cc27a77c7537fb684179c81852b08fc386c92400ea0c 2024-04-05T03:06:15.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:14 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:15.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:14 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[74779]: logger=settings t=2024-04-05T03:06:14.661994035Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:06:15.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:14 smithi151 podman[74797]: 2024-04-05 03:06:14.688555942 +0000 UTC m=+0.019556125 container died 30170eec85dcf2157dd9cc27a77c7537fb684179c81852b08fc386c92400ea0c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.buildah.version=1.32.0, release=239.1651231664, com.redhat.component=ubi8-container, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T03:06:15.525 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:15 smithi151 podman[74797]: 2024-04-05 03:06:15.269718011 +0000 UTC m=+0.600718172 container remove 30170eec85dcf2157dd9cc27a77c7537fb684179c81852b08fc386c92400ea0c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, distribution-scope=public, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, version=8.5) 2024-04-05T03:06:15.525 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:15 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:06:15.525 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:15 smithi151 systemd[1]: Stopping Ceph node-exporter.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:15.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:15 smithi151 ceph-mon[32789]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:15.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:15 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1951814377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:15.525 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:15 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:15.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:15 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:15.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[29244]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:15.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1951814377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:15.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:15.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:15.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[40008]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:15.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1951814377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:15.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:15.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:15 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:15.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:15 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:06:15.778 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:15 smithi151 podman[74955]: 2024-04-05 03:06:15.625021664 +0000 UTC m=+0.122551228 container died 958473fd6ad3e1775df94cc8e304fe295b80304a55bc00964c0ea1ad62a50afa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:06:16.278 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:15 smithi151 podman[74955]: 2024-04-05 03:06:15.989470301 +0000 UTC m=+0.486999873 container cleanup 958473fd6ad3e1775df94cc8e304fe295b80304a55bc00964c0ea1ad62a50afa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:06:16.278 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:15 smithi151 bash[74955]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b 2024-04-05T03:06:16.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-mon[32789]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-05T03:06:16.657 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-mon[32789]: Reconfiguring daemon node-exporter.b on smithi151 2024-04-05T03:06:16.657 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 podman[74971]: 2024-04-05 03:06:16.290636376 +0000 UTC m=+0.669957669 container remove 958473fd6ad3e1775df94cc8e304fe295b80304a55bc00964c0ea1ad62a50afa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:06:16.657 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:06:16.657 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-05T03:06:16.658 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 systemd[1]: Stopped Ceph node-exporter.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:16.658 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.b.service: Consumed 2.012s CPU time. 2024-04-05T03:06:16.658 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 systemd[1]: Starting Ceph node-exporter.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:16.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:16 smithi033 ceph-mon[29244]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-05T03:06:16.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:16 smithi033 ceph-mon[29244]: Reconfiguring daemon node-exporter.b on smithi151 2024-04-05T03:06:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:16 smithi033 ceph-mon[40008]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-05T03:06:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:16 smithi033 ceph-mon[40008]: Reconfiguring daemon node-exporter.b on smithi151 2024-04-05T03:06:16.985 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 podman[75066]: 2024-04-05 03:06:16.658188906 +0000 UTC m=+0.108123760 container create 18b78d165e692025c44126f37f53bc053d3dddf0b99f87a2d995bf6b390cf39a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 podman[75066]: 2024-04-05 03:06:16.56759983 +0000 UTC m=+0.017534685 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 podman[75066]: 2024-04-05 03:06:16.894397641 +0000 UTC m=+0.344332507 container init 18b78d165e692025c44126f37f53bc053d3dddf0b99f87a2d995bf6b390cf39a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 podman[75066]: 2024-04-05 03:06:16.897235372 +0000 UTC m=+0.347170236 container start 18b78d165e692025c44126f37f53bc053d3dddf0b99f87a2d995bf6b390cf39a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.900Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.900Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.900Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.900Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T03:06:16.986 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T03:06:16.987 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T03:06:16.988 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T03:06:16.989 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T03:06:16.990 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T03:06:16.990 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T03:06:16.990 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b[75080]: ts=2024-04-05T03:06:16.901Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T03:06:16.990 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 bash[75066]: 18b78d165e692025c44126f37f53bc053d3dddf0b99f87a2d995bf6b390cf39a 2024-04-05T03:06:17.278 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:06:16 smithi151 systemd[1]: Started Ceph node-exporter.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:17.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/263073578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/263073578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:17.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:06:17.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:17.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:06:17.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:17 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/263073578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:06:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:17 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[29244]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[29244]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[29244]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[29244]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[29244]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[40008]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[40008]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[40008]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[40008]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T03:06:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:18 smithi033 ceph-mon[40008]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T03:06:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:18 smithi151 ceph-mon[32789]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T03:06:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:18 smithi151 ceph-mon[32789]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:18 smithi151 ceph-mon[32789]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T03:06:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:18 smithi151 ceph-mon[32789]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T03:06:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:18 smithi151 ceph-mon[32789]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T03:06:18.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:06:19.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[40008]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:19.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3636434665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[29244]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:19.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:19 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3636434665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:19 smithi151 ceph-mon[32789]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:19 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:19 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:19 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:19 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:06:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:19 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:19 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3636434665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:20.596 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:20 smithi033 ceph-mon[40008]: Upgrade: Updating mgr.y 2024-04-05T03:06:20.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:20 smithi033 ceph-mon[40008]: Deploying daemon mgr.y on smithi033 2024-04-05T03:06:20.597 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:20 smithi033 ceph-mon[29244]: Upgrade: Updating mgr.y 2024-04-05T03:06:20.597 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:20 smithi033 ceph-mon[29244]: Deploying daemon mgr.y on smithi033 2024-04-05T03:06:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:20 smithi151 ceph-mon[32789]: Upgrade: Updating mgr.y 2024-04-05T03:06:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:20 smithi151 ceph-mon[32789]: Deploying daemon mgr.y on smithi033 2024-04-05T03:06:20.908 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:20 smithi033 systemd[1]: Stopping Ceph mgr.y for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:21.202 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:20 smithi033 podman[88385]: 2024-04-05 03:06:20.90931307 +0000 UTC m=+0.127556919 container died 46497f82f21195574557661ee342a21224c8c788f25c2ab38c344f683e325ea8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240403, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-05T03:06:21.203 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 podman[88385]: 2024-04-05 03:06:21.203592651 +0000 UTC m=+0.421836545 container cleanup 46497f82f21195574557661ee342a21224c8c788f25c2ab38c344f683e325ea8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240403) 2024-04-05T03:06:21.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:21 smithi033 ceph-mon[29244]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:21.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:21 smithi033 ceph-mon[40008]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:21.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 bash[88385]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y 2024-04-05T03:06:21.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 podman[88399]: 2024-04-05 03:06:21.379918009 +0000 UTC m=+0.469576857 container remove 46497f82f21195574557661ee342a21224c8c788f25c2ab38c344f683e325ea8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-04-05T03:06:21.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:06:21.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:21 smithi151 ceph-mon[32789]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:21.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:21 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:06:21] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2024-04-05T03:06:21.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service: Failed with result 'exit-code'. 2024-04-05T03:06:21.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 systemd[1]: Stopped Ceph mgr.y for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:21.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service: Consumed 1min 1.512s CPU time. 2024-04-05T03:06:21.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:21 smithi033 systemd[1]: Starting Ceph mgr.y for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:22.366 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 podman[88551]: 2024-04-05 03:06:21.974801598 +0000 UTC m=+0.017863721 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:06:22.366 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 podman[88551]: 2024-04-05 03:06:22.072709579 +0000 UTC m=+0.115771689 container create a6e11bc7dc6c105c336170436d37ec1f1e4d4ab7ac7154080371b6bc1da59077 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-05T03:06:22.366 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 podman[88551]: 2024-04-05 03:06:22.27552588 +0000 UTC m=+0.318587992 container init a6e11bc7dc6c105c336170436d37ec1f1e4d4ab7ac7154080371b6bc1da59077 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, CEPH_POINT_RELEASE=) 2024-04-05T03:06:22.366 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 podman[88551]: 2024-04-05 03:06:22.278143897 +0000 UTC m=+0.321206011 container start a6e11bc7dc6c105c336170436d37ec1f1e4d4ab7ac7154080371b6bc1da59077 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-05T03:06:22.367 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 ceph-mgr[88570]: -- 172.21.15.33:0/2381455994 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a6d48a25a0 con 0x55a6d4876800 2024-04-05T03:06:22.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:22 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4059298234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:22 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4059298234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:22.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 bash[88551]: a6e11bc7dc6c105c336170436d37ec1f1e4d4ab7ac7154080371b6bc1da59077 2024-04-05T03:06:22.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 systemd[1]: Started Ceph mgr.y for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:22 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4059298234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:23.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:22.850+0000 7f885326c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:06:23.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:22 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:22.953+0000 7f885326c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[29244]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[40008]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:23.529 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:23 smithi151 ceph-mon[32789]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:23 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:23 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:23 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:23.927 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:23.656+0000 7f885326c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:06:23.927 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:23.756+0000 7f885326c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:06:24.225 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:23 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:23.927+0000 7f885326c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:06:24.225 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:24.029+0000 7f885326c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:06:24.226 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:24.119+0000 7f885326c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:06:24.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/61399338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:24.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/61399338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:24.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:24.401+0000 7f885326c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:06:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:24 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/61399338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:24.930 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:24.521+0000 7f885326c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:06:24.930 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:24.633+0000 7f885326c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:06:24.930 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:24.741+0000 7f885326c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:06:25.195 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:24.930+0000 7f885326c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:06:25.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:25 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-05T03:06:25.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:25 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:25.697 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:25 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:25.697 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:25 smithi151 ceph-mon[32789]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:25.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:25 smithi033 ceph-mon[29244]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:25.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:25 smithi033 ceph-mon[40008]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:26.100 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:25 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:25.963+0000 7f885326c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:06:26.100 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:26.100+0000 7f885326c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:06:26.585 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:26 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:26.585 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:26 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.585 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:26 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.585 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:26 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.585 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:26 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.586 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 podman[75651]: 2024-04-05 03:06:26.200247169 +0000 UTC m=+0.019479873 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:06:26.586 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 podman[75651]: 2024-04-05 03:06:26.33425918 +0000 UTC m=+0.153491883 container create b088189e61c48b9644424ae0ca2e2965a020bf48dc183e63bca62361f9249644 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., version=8.5, maintainer=Paul Cuzner , name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T03:06:26.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:26.468+0000 7f885326c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:06:26.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:26.567+0000 7f885326c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:27.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 podman[75651]: 2024-04-05 03:06:26.586444873 +0000 UTC m=+0.405677577 container init b088189e61c48b9644424ae0ca2e2965a020bf48dc183e63bca62361f9249644 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, description=Ceph Grafana Container, release=239.1651231664, io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.buildah.version=1.32.0, architecture=x86_64, distribution-scope=public, name=ubi8, vendor=Red Hat, Inc., vcs-type=git, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897) 2024-04-05T03:06:27.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 podman[75651]: 2024-04-05 03:06:26.589188133 +0000 UTC m=+0.408420848 container start b088189e61c48b9644424ae0ca2e2965a020bf48dc183e63bca62361f9249644 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, vcs-type=git, com.redhat.component=ubi8-container, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, description=Ceph Grafana Container) 2024-04-05T03:06:27.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 bash[75651]: b088189e61c48b9644424ae0ca2e2965a020bf48dc183e63bca62361f9249644 2024-04-05T03:06:27.029 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:27.029 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[75666]: logger=settings t=2024-04-05T03:06:26.742921535Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:06:27.029 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:26 smithi151 podman[75683]: 2024-04-05 03:06:26.770136006 +0000 UTC m=+0.020074538 container died b088189e61c48b9644424ae0ca2e2965a020bf48dc183e63bca62361f9249644 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, release=239.1651231664, com.redhat.component=ubi8-container, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, distribution-scope=public) 2024-04-05T03:06:27.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:26.792+0000 7f885326c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:06:27.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:26 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:26.996+0000 7f885326c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:06:27.513 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[29244]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:27.513 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3886065801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:27.513 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:27.513 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:27.513 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:27.312+0000 7f885326c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:06:27.514 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[40008]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:27.514 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3886065801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:27.514 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:27.514 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:27.634 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:27 smithi151 ceph-mon[32789]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:27.634 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:27 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3886065801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:27.635 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:27 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:27.635 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:27 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:27.635 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:27 smithi151 podman[75683]: 2024-04-05 03:06:27.415368793 +0000 UTC m=+0.665307325 container remove b088189e61c48b9644424ae0ca2e2965a020bf48dc183e63bca62361f9249644 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vcs-type=git, io.openshift.tags=base rhel8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, version=8.5, io.buildah.version=1.32.0, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, name=ubi8, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., maintainer=Paul Cuzner , description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-05T03:06:27.635 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:27 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:06:27.862 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:27.592+0000 7f885326c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:06:27.862 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:27 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:27.861+0000 7f885326c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:06:28.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:27 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:06:28.867 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:28 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:28.749+0000 7f885326c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:06:29.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:06:29.457 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:29.367+0000 7f885326c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:06:29.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-mon[29244]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:29.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1370635825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:29.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-mon[40008]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:29.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1370635825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:29.714 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:29.475+0000 7f885326c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:06:29.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:29 smithi151 ceph-mon[32789]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:29.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:29 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1370635825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:29.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:29.713+0000 7f885326c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:06:29.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:29] ENGINE Bus STARTING 2024-04-05T03:06:29.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: CherryPy Checker: 2024-04-05T03:06:29.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: The Application mounted at '' has an empty config. 2024-04-05T03:06:29.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:29.981 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:29] ENGINE Serving on http://:::9283 2024-04-05T03:06:29.981 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:29] ENGINE Bus STARTED 2024-04-05T03:06:30.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[40008]: Standby manager daemon y restarted 2024-04-05T03:06:30.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[40008]: Standby manager daemon y started 2024-04-05T03:06:30.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T03:06:30.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:06:30.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T03:06:30.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:06:30.567 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[29244]: Standby manager daemon y restarted 2024-04-05T03:06:30.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[29244]: Standby manager daemon y started 2024-04-05T03:06:30.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T03:06:30.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:06:30.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T03:06:30.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:06:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-mon[32789]: Standby manager daemon y restarted 2024-04-05T03:06:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-mon[32789]: Standby manager daemon y started 2024-04-05T03:06:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T03:06:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:06:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T03:06:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:06:30.826 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:30] ENGINE Bus STOPPING 2024-04-05T03:06:30.826 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:06:30.826 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:30] ENGINE Bus STOPPED 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:30] ENGINE Bus STARTING 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.918+0000 7f87aff01700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.918+0000 7f87b3f09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.918+0000 7f87b3f09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.918+0000 7f87b3f09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.918+0000 7f87b3f09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.918+0000 7f87b3f09700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.918+0000 7f87aff01700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.919+0000 7f87aff01700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.919+0000 7f87aff01700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.232 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:06:30.919+0000 7f87aff01700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:06:31.232 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:30] ENGINE Serving on http://:::9283 2024-04-05T03:06:31.232 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:06:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:06:30] ENGINE Bus STARTED 2024-04-05T03:06:31.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ignoring --setuser ceph since I am not root 2024-04-05T03:06:31.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ignoring --setgroup ceph since I am not root 2024-04-05T03:06:31.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:30 smithi151 ceph-mgr[66213]: -- 172.21.15.151:0/2903091395 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559c5f3585a0 con 0x559c5f32c800 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: mgrmap e26: x(active, since 64s), standbys: y 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T03:06:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25258 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: mgrmap e27: y(active, starting, since 0.00685949s) 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:06:31.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: Manager daemon y is now available 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:06:31.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: mgrmap e26: x(active, since 64s), standbys: y 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25258 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-05T03:06:31.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: mgrmap e27: y(active, starting, since 0.00685949s) 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:06:31.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:06:31.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:06:31.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:06:31.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: Manager daemon y is now available 2024-04-05T03:06:31.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:31.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:31.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:06:31.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:31 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:06:31.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:31.381+0000 7f93ef74a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:06:31.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:31.479+0000 7f93ef74a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:06:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: mgrmap e26: x(active, since 64s), standbys: y 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 172.21.15.151:0/336657541' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25258 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: mgrmap e27: y(active, starting, since 0.00685949s) 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:06:31.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:06:31.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:06:31.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:06:31.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: Manager daemon y is now available 2024-04-05T03:06:31.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:31.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:31.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:06:31.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:31 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:06:32.271 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:32.176+0000 7f93ef74a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:06:32.271 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:32.271+0000 7f93ef74a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:06:32.525 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:32.417+0000 7f93ef74a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:32.525+0000 7f93ef74a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: [05/Apr/2024:03:06:31] ENGINE Bus STARTING 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: [05/Apr/2024:03:06:31] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3663158556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: [05/Apr/2024:03:06:31] ENGINE Serving on http://172.21.15.33:8765 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: [05/Apr/2024:03:06:31] ENGINE Bus STARTED 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: mgrmap e28: y(active, since 1.009s) 2024-04-05T03:06:32.526 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-mon[32789]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: [05/Apr/2024:03:06:31] ENGINE Bus STARTING 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: [05/Apr/2024:03:06:31] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3663158556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: [05/Apr/2024:03:06:31] ENGINE Serving on http://172.21.15.33:8765 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: [05/Apr/2024:03:06:31] ENGINE Bus STARTED 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: mgrmap e28: y(active, since 1.009s) 2024-04-05T03:06:32.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[40008]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: [05/Apr/2024:03:06:31] ENGINE Bus STARTING 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: [05/Apr/2024:03:06:31] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3663158556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: [05/Apr/2024:03:06:31] ENGINE Serving on http://172.21.15.33:8765 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: [05/Apr/2024:03:06:31] ENGINE Bus STARTED 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: mgrmap e28: y(active, since 1.009s) 2024-04-05T03:06:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:32 smithi033 ceph-mon[29244]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:32.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:32.628+0000 7f93ef74a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:06:33.107 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:32 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:32.907+0000 7f93ef74a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:06:33.107 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:33 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:33.019+0000 7f93ef74a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:06:33.426 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:33 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:33.107+0000 7f93ef74a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:06:33.426 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:33 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:33.210+0000 7f93ef74a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:06:33.426 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:33 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:33.426+0000 7f93ef74a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:06:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:33 smithi033 ceph-mon[29244]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T03:06:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:33 smithi033 ceph-mon[29244]: Cluster is now healthy 2024-04-05T03:06:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:33 smithi033 ceph-mon[29244]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:33 smithi033 ceph-mon[40008]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T03:06:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:33 smithi033 ceph-mon[40008]: Cluster is now healthy 2024-04-05T03:06:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:33 smithi033 ceph-mon[40008]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:33.732 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:33 smithi151 ceph-mon[32789]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T03:06:33.732 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:33 smithi151 ceph-mon[32789]: Cluster is now healthy 2024-04-05T03:06:33.732 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:33 smithi151 ceph-mon[32789]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:34.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:34 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:34.451+0000 7f93ef74a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:06:34.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:34 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:34.586+0000 7f93ef74a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:06:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:34 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3806474116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:34 smithi033 ceph-mon[29244]: mgrmap e29: y(active, since 3s) 2024-04-05T03:06:35.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:34 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:34 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3806474116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:34 smithi033 ceph-mon[40008]: mgrmap e29: y(active, since 3s) 2024-04-05T03:06:35.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:34 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:35.258 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:34 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3806474116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:35.258 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:34 smithi151 ceph-mon[32789]: mgrmap e29: y(active, since 3s) 2024-04-05T03:06:35.258 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:34 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:35.259 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:34 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:34.935+0000 7f93ef74a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:06:35.259 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:35.029+0000 7f93ef74a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:06:35.259 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:35.259+0000 7f93ef74a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:06:35.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:35.461+0000 7f93ef74a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:06:36.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:35.769+0000 7f93ef74a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:06:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-mon[32789]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:35 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[40008]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:36.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[29244]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:36.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:35 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:36.309 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:36 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:36.050+0000 7f93ef74a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:06:36.309 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:36 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:36.309+0000 7f93ef74a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:06:37.138 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:36 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.138 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:36 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.138 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:36 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/121434698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:37.138 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:36 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.138 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:36 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/121434698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/121434698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:36 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:37.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:37.138+0000 7f93ef74a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:06:38.016 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:37 smithi151 ceph-mon[32789]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:38.016 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:37.721+0000 7f93ef74a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:06:38.017 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:37.825+0000 7f93ef74a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:06:38.017 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:37 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-05T03:06:38.017 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:37 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:38.017 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:37 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:38.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:37 smithi033 ceph-mon[29244]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:38.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:37 smithi033 ceph-mon[40008]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:38.274 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 podman[76581]: 2024-04-05 03:06:37.92054474 +0000 UTC m=+0.020523823 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:06:38.274 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 podman[76581]: 2024-04-05 03:06:38.023920812 +0000 UTC m=+0.123899873 container create 1e9b02919fb76e649e34988d47ef6a2700db0edfc97eba2bb0dff62b5048d4e0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, release=239.1651231664, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, version=8.5, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public) 2024-04-05T03:06:38.274 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:38.069+0000 7f93ef74a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:06:38.274 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:06:38] ENGINE Bus STARTING 2024-04-05T03:06:38.274 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: CherryPy Checker: 2024-04-05T03:06:38.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: The Application mounted at '' has an empty config. 2024-04-05T03:06:38.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: 2024-04-05T03:06:38.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:06:38] ENGINE Serving on http://:::9283 2024-04-05T03:06:38.275 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: [05/Apr/2024:03:06:38] ENGINE Bus STARTED 2024-04-05T03:06:38.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 podman[76581]: 2024-04-05 03:06:38.275330014 +0000 UTC m=+0.375309075 container init 1e9b02919fb76e649e34988d47ef6a2700db0edfc97eba2bb0dff62b5048d4e0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, release=239.1651231664, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , version=8.5, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, name=ubi8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=) 2024-04-05T03:06:38.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 podman[76581]: 2024-04-05 03:06:38.27858218 +0000 UTC m=+0.378561242 container start 1e9b02919fb76e649e34988d47ef6a2700db0edfc97eba2bb0dff62b5048d4e0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.buildah.version=1.32.0, name=ubi8, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-05T03:06:38.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 bash[76581]: 1e9b02919fb76e649e34988d47ef6a2700db0edfc97eba2bb0dff62b5048d4e0 2024-04-05T03:06:38.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:38.529 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[76623]: logger=settings t=2024-04-05T03:06:38.428691366Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:06:38.529 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:38 smithi151 podman[76641]: 2024-04-05 03:06:38.454675963 +0000 UTC m=+0.019108178 container died 1e9b02919fb76e649e34988d47ef6a2700db0edfc97eba2bb0dff62b5048d4e0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., release=239.1651231664, architecture=x86_64, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, com.redhat.component=ubi8-container, description=Ceph Grafana Container, maintainer=Paul Cuzner ) 2024-04-05T03:06:38.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[29244]: Standby manager daemon x started 2024-04-05T03:06:38.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3438883567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[40008]: Standby manager daemon x started 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:06:38.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:38 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3438883567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:39.025 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-mon[32789]: Standby manager daemon x started 2024-04-05T03:06:39.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:06:39.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:06:39.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:06:39.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/2029188954' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:06:39.026 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:38 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3438883567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:39.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:39 smithi151 podman[76641]: 2024-04-05 03:06:39.094250886 +0000 UTC m=+0.658683095 container remove 1e9b02919fb76e649e34988d47ef6a2700db0edfc97eba2bb0dff62b5048d4e0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.expose-services=, io.buildah.version=1.32.0, release=239.1651231664, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., version=8.5, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-05T03:06:39.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:39 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:06:39.668 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:39 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:06:40.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:39 smithi151 ceph-mon[32789]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:06:40.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:39 smithi151 ceph-mon[32789]: mgrmap e30: y(active, since 8s), standbys: x 2024-04-05T03:06:40.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:39 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:06:40.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:39 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:40.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:39 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:40.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:39 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:40.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:39 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[40008]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[40008]: mgrmap e30: y(active, since 8s), standbys: x 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:40.124 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[29244]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:06:40.125 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[29244]: mgrmap e30: y(active, since 8s), standbys: x 2024-04-05T03:06:40.125 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:06:40.125 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:40.125 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:40.125 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:40.125 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:39 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2969887285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:41 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2969887285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:41.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[66209]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:06:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T03:06:41.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:06:41.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:06:41.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:06:41.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:06:41.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:41.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:41 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2969887285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:42.377 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:42 smithi151 ceph-mon[32789]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:06:42.377 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:42 smithi151 ceph-mon[32789]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:06:42.377 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:42 smithi151 ceph-mon[32789]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:06:42.377 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:42 smithi151 ceph-mon[32789]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:06:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[29244]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:06:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[29244]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:06:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[29244]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:06:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[29244]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:06:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[40008]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:06:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[40008]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T03:06:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[40008]: Reconfiguring daemon prometheus.a on smithi151 2024-04-05T03:06:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:42 smithi033 ceph-mon[40008]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:06:44.646 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:44 smithi151 ceph-mon[32789]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-05T03:06:44.646 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:44 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1732549190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:44.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:44 smithi033 ceph-mon[40008]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-05T03:06:44.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:44 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1732549190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:44.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:44 smithi033 ceph-mon[29244]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-05T03:06:44.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:44 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1732549190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:45.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:44 smithi151 systemd[1]: Stopping Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:45.318 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.034Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:06:45.318 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.034Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T03:06:45.318 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T03:06:45.318 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T03:06:45.318 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[74327]: ts=2024-04-05T03:06:45.035Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T03:06:45.319 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 podman[77750]: 2024-04-05 03:06:45.044758566 +0000 UTC m=+0.103710068 container died 2f71aa61c3374a18ea5c8c8ff9f63fbec73d3d48e8ab14f6e19e7306d0cb5c5f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:45.684 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 podman[77750]: 2024-04-05 03:06:45.417522713 +0000 UTC m=+0.476474204 container cleanup 2f71aa61c3374a18ea5c8c8ff9f63fbec73d3d48e8ab14f6e19e7306d0cb5c5f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:45.684 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 bash[77750]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T03:06:46.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 podman[77764]: 2024-04-05 03:06:45.685415167 +0000 UTC m=+0.641828615 container remove 2f71aa61c3374a18ea5c8c8ff9f63fbec73d3d48e8ab14f6e19e7306d0cb5c5f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:46.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-05T03:06:46.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 systemd[1]: Stopped Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:46.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service: Consumed 1.079s CPU time. 2024-04-05T03:06:46.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:45 smithi151 systemd[1]: Starting Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:46.305 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 podman[77856]: 2024-04-05 03:06:45.949521228 +0000 UTC m=+0.018759860 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T03:06:46.305 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 podman[77856]: 2024-04-05 03:06:46.069974654 +0000 UTC m=+0.139213327 container create c6c02f0b66d84945d2594a63bc6d0f5b732171ed14c8458fc610f08aa444f81a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:46.305 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 podman[77856]: 2024-04-05 03:06:46.288755475 +0000 UTC m=+0.357994106 container init c6c02f0b66d84945d2594a63bc6d0f5b732171ed14c8458fc610f08aa444f81a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:46.306 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 podman[77856]: 2024-04-05 03:06:46.29134136 +0000 UTC m=+0.360579998 container start c6c02f0b66d84945d2594a63bc6d0f5b732171ed14c8458fc610f08aa444f81a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:06:46.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[29244]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:46.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3833651418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:46.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[40008]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:46.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:46.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:46.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:46 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3833651418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-mon[32789]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3833651418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:46.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.315Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.315Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.315Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi151 (none))" 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.315Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.315Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.317Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.317Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.319Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.319Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.321Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.321Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.462µs 2024-04-05T03:06:46.779 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.321Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.321Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.335Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.336Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.338Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.338Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.47µs wal_replay_duration=17.308874ms wbl_replay_duration=182ns total_replay_duration=17.365174ms 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.341Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.341Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.341Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.356Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.215138ms db_storage=836ns remote_storage=1.016µs web_handler=361ns query_engine=638ns scrape=392.013µs scrape_sd=86.303µs notify=10.253µs notify_sd=9.97µs rules=14.494205ms tracing=5.233µs 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.356Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:06:46.356Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 bash[77856]: c6c02f0b66d84945d2594a63bc6d0f5b732171ed14c8458fc610f08aa444f81a 2024-04-05T03:06:46.780 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:06:46 smithi151 systemd[1]: Started Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[29244]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[29244]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[40008]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[40008]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:47.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:47 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:47 smithi151 ceph-mon[32789]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:47 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:47 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:47 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:47 smithi151 ceph-mon[32789]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T03:06:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:47 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:48.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:48 smithi151 ceph-mon[32789]: Upgrade: Updating mgr.x 2024-04-05T03:06:48.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:48 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:48.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:48 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:06:48.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:48 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:48.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:48 smithi151 ceph-mon[32789]: Deploying daemon mgr.x on smithi151 2024-04-05T03:06:48.686 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:48 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3057486382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[29244]: Upgrade: Updating mgr.x 2024-04-05T03:06:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:06:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[29244]: Deploying daemon mgr.x on smithi151 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3057486382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[40008]: Upgrade: Updating mgr.x 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[40008]: Deploying daemon mgr.x on smithi151 2024-04-05T03:06:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:48 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3057486382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:49.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:49 smithi033 ceph-mon[29244]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:49.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:49 smithi033 ceph-mon[40008]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:49.731 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:49 smithi151 ceph-mon[32789]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-05T03:06:49.731 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:49 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-05T03:06:49.731 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:49 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:49.731 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:49 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:50.022 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:49 smithi151 systemd[1]: Stopping Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:50.023 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:49 smithi151 podman[78192]: 2024-04-05 03:06:49.646349797 +0000 UTC m=+0.019436641 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:06:50.023 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:49 smithi151 podman[78192]: 2024-04-05 03:06:49.779389589 +0000 UTC m=+0.152476417 container create c865421ffc4caefece86546553121a452b649d65b729888201a8f54e4af570fe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, description=Ceph Grafana Container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, name=ubi8) 2024-04-05T03:06:50.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 podman[78192]: 2024-04-05 03:06:50.023762279 +0000 UTC m=+0.396849108 container init c865421ffc4caefece86546553121a452b649d65b729888201a8f54e4af570fe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, release=239.1651231664, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, vendor=Red Hat, Inc., name=ubi8, io.openshift.expose-services=, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897) 2024-04-05T03:06:50.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 podman[78192]: 2024-04-05 03:06:50.026808173 +0000 UTC m=+0.399895010 container start c865421ffc4caefece86546553121a452b649d65b729888201a8f54e4af570fe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, architecture=x86_64, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, name=ubi8, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, vcs-type=git, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-04-05T03:06:50.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 bash[78192]: c865421ffc4caefece86546553121a452b649d65b729888201a8f54e4af570fe 2024-04-05T03:06:50.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:50.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[78252]: logger=settings t=2024-04-05T03:06:50.18683939Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:06:50.279 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 podman[78283]: 2024-04-05 03:06:50.216555268 +0000 UTC m=+0.022452900 container died c865421ffc4caefece86546553121a452b649d65b729888201a8f54e4af570fe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.expose-services=, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, maintainer=Paul Cuzner , release=239.1651231664, version=8.5, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.openshift.tags=base rhel8) 2024-04-05T03:06:50.769 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 podman[78283]: 2024-04-05 03:06:50.768652063 +0000 UTC m=+0.574549673 container remove c865421ffc4caefece86546553121a452b649d65b729888201a8f54e4af570fe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, description=Ceph Grafana Container, maintainer=Paul Cuzner ) 2024-04-05T03:06:51.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:50 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:06:51.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:50 smithi151 podman[78284]: 2024-04-05 03:06:50.977355789 +0000 UTC m=+0.783253335 container died 80ca6768c17e2b56da0bfb677e527c97f2cfd8cbdf86babe88bde936c79bff9d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-0028525, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:06:51.513 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:06:51 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:06:51.513 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:51 smithi151 ceph-mon[32789]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:51.513 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:51 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1922795399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:51.513 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 podman[78284]: 2024-04-05 03:06:51.312742997 +0000 UTC m=+1.118640528 container cleanup 80ca6768c17e2b56da0bfb677e527c97f2cfd8cbdf86babe88bde936c79bff9d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-0028525, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:06:51.513 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 bash[78284]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x 2024-04-05T03:06:51.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:51 smithi033 ceph-mon[40008]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:51.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:51 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1922795399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:51.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:51 smithi033 ceph-mon[29244]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:51.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:51 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/1922795399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:51.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 podman[78331]: 2024-04-05 03:06:51.514067 +0000 UTC m=+0.537947764 container remove 80ca6768c17e2b56da0bfb677e527c97f2cfd8cbdf86babe88bde936c79bff9d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:06:51.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:06:51.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-05T03:06:51.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 systemd[1]: Stopped Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:51.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service: Consumed 26.902s CPU time. 2024-04-05T03:06:52.235 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:51 smithi151 systemd[1]: Starting Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:06:52.236 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 podman[78473]: 2024-04-05 03:06:52.040406816 +0000 UTC m=+0.018180301 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:06:52.236 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 podman[78473]: 2024-04-05 03:06:52.139973371 +0000 UTC m=+0.117746883 container create 090d83bbd680c504ba3b7e5bc6d6e695e1a3443f2496965cd3363054e6ee3871 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403) 2024-04-05T03:06:52.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 podman[78473]: 2024-04-05 03:06:52.34263225 +0000 UTC m=+0.320405720 container init 090d83bbd680c504ba3b7e5bc6d6e695e1a3443f2496965cd3363054e6ee3871 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, GIT_BRANCH=HEAD) 2024-04-05T03:06:52.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 podman[78473]: 2024-04-05 03:06:52.345183173 +0000 UTC m=+0.322956648 container start 090d83bbd680c504ba3b7e5bc6d6e695e1a3443f2496965cd3363054e6ee3871 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:06:52.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 ceph-mgr[78492]: -- 172.21.15.151:0/3681035129 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558529db25a0 con 0x558529d86c00 2024-04-05T03:06:52.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 bash[78473]: 090d83bbd680c504ba3b7e5bc6d6e695e1a3443f2496965cd3363054e6ee3871 2024-04-05T03:06:52.528 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 systemd[1]: Started Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:06:53.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:52.922+0000 7f8c84d86200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:06:53.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:53.027+0000 7f8c84d86200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:06:53.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-mon[32789]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:53.718 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:53.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.719 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4108896037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:53.969 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:53.718+0000 7f8c84d86200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:06:53.969 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:53.826+0000 7f8c84d86200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:06:53.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[29244]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:53.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:53.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/4108896037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[40008]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:53.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:53 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4108896037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:54.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:53.970+0000 7f8c84d86200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:06:54.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:54.073+0000 7f8c84d86200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:06:54.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:54.162+0000 7f8c84d86200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:06:54.772 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:54.432+0000 7f8c84d86200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:06:54.772 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:54.564+0000 7f8c84d86200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:06:54.772 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:54.669+0000 7f8c84d86200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:06:54.772 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:54.772+0000 7f8c84d86200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:06:55.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:54.961+0000 7f8c84d86200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:06:55.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:55 smithi151 ceph-mon[32789]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:55.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:55 smithi033 ceph-mon[29244]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:55.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:55 smithi033 ceph-mon[40008]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:56.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:56.001+0000 7f8c84d86200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:06:56.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:56.134+0000 7f8c84d86200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:06:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2801226262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:56.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:56.492+0000 7f8c84d86200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:06:56.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:56.591+0000 7f8c84d86200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:06:56.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:56 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2801226262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:56.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:56 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:56.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:56 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:56.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:56 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2801226262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:56.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:56 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:56.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:56 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:57.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:56.806+0000 7f8c84d86200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:06:57.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:57.010+0000 7f8c84d86200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:06:57.593 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:57 smithi151 ceph-mon[32789]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:57.593 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:57.319+0000 7f8c84d86200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:06:57.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:57.593+0000 7f8c84d86200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:06:57.848 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:57.848+0000 7f8c84d86200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:06:57.881 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:57 smithi033 ceph-mon[29244]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:57.882 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:57 smithi033 ceph-mon[40008]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:58.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:58.707+0000 7f8c84d86200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:06:58.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:58 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:58.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:58 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:58.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:58 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2318418135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:58.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:58 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:58.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:58 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:58 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2318418135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:58.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:58 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:58.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:58 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:06:58.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:58 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2318418135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:06:59.438 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:59.296+0000 7f8c84d86200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:06:59.439 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:59.400+0000 7f8c84d86200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:06:59.745 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-mon[32789]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:59.745 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:06:59.637+0000 7f8c84d86200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:06:59.745 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: [05/Apr/2024:03:06:59] ENGINE Bus STARTING 2024-04-05T03:06:59.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:06:59 smithi033 ceph-mon[29244]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:06:59.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:06:59 smithi033 ceph-mon[40008]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:00.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: CherryPy Checker: 2024-04-05T03:07:00.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: The Application mounted at '' has an empty config. 2024-04-05T03:07:00.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:00.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: [05/Apr/2024:03:06:59] ENGINE Serving on http://:::9283 2024-04-05T03:07:00.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:06:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: [05/Apr/2024:03:06:59] ENGINE Bus STARTED 2024-04-05T03:07:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:00 smithi151 ceph-mon[32789]: Standby manager daemon x restarted 2024-04-05T03:07:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:00 smithi151 ceph-mon[32789]: Standby manager daemon x started 2024-04-05T03:07:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:00 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:07:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:00 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:07:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:00 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:07:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:00 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:07:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:00 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3244460653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:00.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:07:00] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[29244]: Standby manager daemon x restarted 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[29244]: Standby manager daemon x started 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[29244]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/3244460653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[40008]: Standby manager daemon x restarted 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[40008]: Standby manager daemon x started 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:07:00.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:07:00.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:07:00.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/221679320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:07:00.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:00 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3244460653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:01.796 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:01 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-05T03:07:01.796 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:01 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:01.796 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:01 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:07:01.796 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:01 smithi151 podman[82248]: 2024-04-05 03:07:01.664354762 +0000 UTC m=+0.016973462 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:07:01.796 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:01 smithi151 podman[82248]: 2024-04-05 03:07:01.797070359 +0000 UTC m=+0.149689057 container create 98533611305f86e5f914f2470abe523c201eedeef5d5859bd757ba440253e3f2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.buildah.version=1.32.0, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, com.redhat.component=ubi8-container, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8) 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: mgrmap e31: y(active, since 29s), standbys: x 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:01.797 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: Upgrade: Setting container_image for all mgr 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-05T03:07:01.798 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:01 smithi151 ceph-mon[32789]: Upgrade: It appears safe to stop mon.a 2024-04-05T03:07:01.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: mgrmap e31: y(active, since 29s), standbys: x 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:01.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: Upgrade: Setting container_image for all mgr 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[29244]: Upgrade: It appears safe to stop mon.a 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: mgrmap e31: y(active, since 29s), standbys: x 2024-04-05T03:07:01.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: Upgrade: Setting container_image for all mgr 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T03:07:01.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-05T03:07:01.906 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T03:07:01.906 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-05T03:07:01.906 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:07:01.906 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-05T03:07:01.906 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:01 smithi033 ceph-mon[40008]: Upgrade: It appears safe to stop mon.a 2024-04-05T03:07:02.115 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 podman[82248]: 2024-04-05 03:07:02.042147124 +0000 UTC m=+0.394765847 container init 98533611305f86e5f914f2470abe523c201eedeef5d5859bd757ba440253e3f2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, name=ubi8, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, architecture=x86_64, distribution-scope=public, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, build-date=2022-04-29T11:48:11.055897) 2024-04-05T03:07:02.115 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 podman[82248]: 2024-04-05 03:07:02.045106909 +0000 UTC m=+0.397725615 container start 98533611305f86e5f914f2470abe523c201eedeef5d5859bd757ba440253e3f2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, name=ubi8, com.redhat.component=ubi8-container, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., release=239.1651231664, vcs-type=git, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , distribution-scope=public, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-05T03:07:02.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 bash[82248]: 98533611305f86e5f914f2470abe523c201eedeef5d5859bd757ba440253e3f2 2024-04-05T03:07:02.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:02.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[82262]: logger=settings t=2024-04-05T03:07:02.19705214Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:07:02.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 podman[82280]: 2024-04-05 03:07:02.222653094 +0000 UTC m=+0.018678861 container died 98533611305f86e5f914f2470abe523c201eedeef5d5859bd757ba440253e3f2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vcs-type=git, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Paul Cuzner , name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vendor=Red Hat, Inc.) 2024-04-05T03:07:03.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 podman[82280]: 2024-04-05 03:07:02.803564365 +0000 UTC m=+0.599590191 container remove 98533611305f86e5f914f2470abe523c201eedeef5d5859bd757ba440253e3f2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, version=8.5, architecture=x86_64, maintainer=Paul Cuzner , io.openshift.expose-services=, vcs-type=git, com.redhat.component=ubi8-container, distribution-scope=public, description=Ceph Grafana Container) 2024-04-05T03:07:03.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:02 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:07:03.278 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:03 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: Upgrade: Updating mon.a 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: Deploying daemon mon.a on smithi033 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[29244]: from='client.? 172.21.15.33:0/2179166079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: Upgrade: Updating mon.a 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: Deploying daemon mon.a on smithi033 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:03.887 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:03 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2179166079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: Upgrade: Updating mon.a 2024-04-05T03:07:04.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' 2024-04-05T03:07:04.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:07:04.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:07:04.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:04.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: Deploying daemon mon.a on smithi033 2024-04-05T03:07:04.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:04.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:03 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2179166079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:04.811 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:04 smithi033 systemd[1]: Stopping Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a[29240]: 2024-04-05T03:07:04.810+0000 7f110fbd8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a[29240]: 2024-04-05T03:07:04.810+0000 7f110fbd8700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:04 smithi033 podman[97537]: 2024-04-05 03:07:04.872521731 +0000 UTC m=+0.186696465 container died e876c4ff5784adc7a7db0a8e79e7bb9aeedd1f4a51fdc3cce2573a612daeaf28 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:04] ENGINE Bus STOPPING 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:04] ENGINE Bus STOPPED 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:04] ENGINE Bus STARTING 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:04] ENGINE Serving on http://:::9283 2024-04-05T03:07:05.153 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:04] ENGINE Bus STARTED 2024-04-05T03:07:05.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:05 smithi033 podman[97537]: 2024-04-05 03:07:05.361576846 +0000 UTC m=+0.675751645 container cleanup e876c4ff5784adc7a7db0a8e79e7bb9aeedd1f4a51fdc3cce2573a612daeaf28 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240403) 2024-04-05T03:07:05.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:05 smithi033 bash[97537]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a 2024-04-05T03:07:05.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:05 smithi033 podman[97562]: 2024-04-05 03:07:05.588135858 +0000 UTC m=+0.712948708 container remove e876c4ff5784adc7a7db0a8e79e7bb9aeedd1f4a51fdc3cce2573a612daeaf28 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403) 2024-04-05T03:07:05.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:05 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-05T03:07:05.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:05 smithi033 systemd[1]: Stopped Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:05.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:05 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service: Consumed 12.379s CPU time. 2024-04-05T03:07:06.219 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:05 smithi033 systemd[1]: Starting Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:07:06.219 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 podman[97719]: 2024-04-05 03:07:06.139652181 +0000 UTC m=+0.099327851 container create 7657e117c64ff20299b2edb0722221d7da3578304e1b51824221e624a415d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-05T03:07:06.219 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 podman[97719]: 2024-04-05 03:07:06.058486821 +0000 UTC m=+0.018162507 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:07:06.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 podman[97719]: 2024-04-05 03:07:06.334085634 +0000 UTC m=+0.293761317 container init 7657e117c64ff20299b2edb0722221d7da3578304e1b51824221e624a415d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 podman[97719]: 2024-04-05 03:07:06.337409456 +0000 UTC m=+0.297085129 container start 7657e117c64ff20299b2edb0722221d7da3578304e1b51824221e624a415d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev), process ceph-mon, pid 2 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: pidfile_write: ignore empty --pid-file 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: load: jerasure load: lrc 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: RocksDB version: 7.9.2 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Git sha 0 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Compile date 2024-04-04 02:01:22 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: DB SUMMARY 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: DB Session ID: KLBZ4M7Q3BWVB6TP4RIN 2024-04-05T03:07:06.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: CURRENT file: CURRENT 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: MANIFEST file: MANIFEST-000015 size: 971 Bytes 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000030.sst 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000028.log size: 2576948 ; 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.error_if_exists: 0 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.create_if_missing: 0 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.paranoid_checks: 1 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.env: 0x55c2571ebc60 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.fs: PosixFileSystem 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.info_log: 0x55c2593c5ae0 2024-04-05T03:07:06.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T03:07:06.483 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.statistics: (nil) 2024-04-05T03:07:06.484 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.use_fsync: 0 2024-04-05T03:07:06.484 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_log_file_size: 0 2024-04-05T03:07:06.484 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T03:07:06.484 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T03:07:06.484 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T03:07:06.484 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.allow_fallocate: 1 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.use_direct_reads: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.db_log_dir: 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.wal_dir: 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T03:07:06.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.write_buffer_manager: 0x55c259398820 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T03:07:06.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.unordered_write: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.row_cache: None 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.wal_filter: None 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.two_write_queues: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.wal_compression: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.atomic_flush: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T03:07:06.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.log_readahead_size: 0 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_background_jobs: 2 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_background_compactions: -1 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_subcompactions: 1 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T03:07:06.488 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_open_files: -1 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_background_flushes: -1 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Compression algorithms supported: 2024-04-05T03:07:06.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kZSTD supported: 0 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kXpressCompression supported: 0 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kZlibCompression supported: 1 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kSnappyCompression supported: 1 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kLZ4Compression supported: 1 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: kBZip2Compression supported: 0 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.merge_operator: 2024-04-05T03:07:06.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_filter: None 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c2593c5be0) 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_type: 0 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_index_type: 0 2024-04-05T03:07:06.491 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_shortening: 1 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: checksum: 4 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: no_block_cache: 0 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache: 0x55c2593c9090 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_options: 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: capacity : 536870912 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: num_shard_bits : 4 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: strict_capacity_limit : 0 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_compressed: (nil) 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: persistent_cache: (nil) 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size: 4096 2024-04-05T03:07:06.492 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size_deviation: 10 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_restart_interval: 16 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_block_restart_interval: 1 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: metadata_block_size: 4096 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: partition_filters: 0 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: use_delta_encoding: 1 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: filter_policy: bloomfilter 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: whole_key_filtering: 1 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: verify_compression: 0 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: format_version: 5 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_index_compression: 1 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_align: 0 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_auto_readahead_size: 262144 2024-04-05T03:07:06.493 INFO:journalctl@ceph.mon.a.smithi033.stdout: prepopulate_block_cache: 0 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout: initial_auto_readahead_size: 8192 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout: num_file_reads_for_auto_readahead: 2 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression: NoCompression 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.num_levels: 7 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T03:07:06.494 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T03:07:06.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T03:07:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.arena_block_size: 1048576 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T03:07:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.table_properties_collectors: 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.inplace_update_support: 0 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.bloom_locality: 0 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.max_successive_merges: 0 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T03:07:06.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.ttl: 2592000 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.enable_blob_files: false 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.min_blob_size: 0 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-05T03:07:06.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.blob_file_starting_level: 0 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 10709, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d841786c-f4a7-4c2d-a11f-c54511b96e84 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286426366177, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286426375239, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2348256, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10710, "largest_seqno": 11831, "table_properties": {"data_size": 2342325, "index_size": 3568, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 12123, "raw_average_key_size": 23, "raw_value_size": 2331043, "raw_average_value_size": 4457, "num_data_blocks": 164, "num_entries": 523, "num_filter_entries": 523, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712286426, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d841786c-f4a7-4c2d-a11f-c54511b96e84", "db_session_id": "KLBZ4M7Q3BWVB6TP4RIN", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286426375365, "job": 1, "event": "recovery_finished"} 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c2594ba000 2024-04-05T03:07:06.500 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: DB pointer 0x55c2594a4000 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** DB Stats ** 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: L0 1/0 2.24 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-05T03:07:06.501 INFO:journalctl@ceph.mon.a.smithi033.stdout: L6 1/0 6.28 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: Sum 2/0 8.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 517.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T03:07:06.502 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative compaction: 0.00 GB write, 170.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval compaction: 0.00 GB write, 170.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: Block cache BinnedLRUCache@0x55c2593c9090#2 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: starting mon.a rank 0 at public addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] at bind addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???) e3 preinit fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???).mds e8 new map 2024-04-05T03:07:06.503 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???).mds e8 print_map 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: e8 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: legacy client fscid: 1 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: Filesystem 'foo' (1) 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: fs_name foo 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: epoch 7 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: created 2024-04-05T03:02:48.450117+0000 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: modified 2024-04-05T03:03:33.860260+0000 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: tableserver 0 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: root 0 2024-04-05T03:07:06.504 INFO:journalctl@ceph.mon.a.smithi033.stdout: session_timeout 60 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: session_autoclose 300 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_file_size 1099511627776 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_xattr_size 65536 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: required_client_features {} 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: last_failure 0 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: last_failure_osd_epoch 0 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_mds 1 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: in 0 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: up {0=14718} 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: failed 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: damaged 2024-04-05T03:07:06.505 INFO:journalctl@ceph.mon.a.smithi033.stdout: stopped 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_pools [4] 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: metadata_pool 3 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: inline_data disabled 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: balancer 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: bal_rank_mask -1 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: standby_count_wanted 1 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: qdb_cluster leader: 0 members: 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: [mds.foo.smithi033.rmzwzs{0:14718} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: Standby daemons: 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout: [mds.foo.smithi151.cxpowy{-1:24535} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T03:07:06.506 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???).osd e87 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-05T03:07:06.507 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???).osd e87 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:07:06.507 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???).osd e87 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:07:06.507 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???).osd e87 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:07:06.507 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 ceph-mon[97738]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-04-05T03:07:06.507 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 bash[97719]: 7657e117c64ff20299b2edb0722221d7da3578304e1b51824221e624a415d490 2024-04-05T03:07:06.507 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:06 smithi033 systemd[1]: Started Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:09.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:07:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:07:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:07:09.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ignoring --setuser ceph since I am not root 2024-04-05T03:07:09.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ignoring --setgroup ceph since I am not root 2024-04-05T03:07:09.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mgr[88570]: -- 172.21.15.33:0/3026463736 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563b6b9965a0 con 0x563b6b96a800 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/3258519851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/4245105634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:09.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: mon.a calling monitor election 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: monmap epoch 3 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: last_changed 2024-04-05T02:57:44.232912+0000 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: min_mon_release 17 (quincy) 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: election_strategy: 1 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:07:09.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: mgrmap e31: y(active, since 38s), standbys: x 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: daemon grafana.a on smithi151 is in unknown state 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: mgrmap e32: y(active, since 38s), standbys: x 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[40008]: from='mgr.25766 ' entity='' 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3258519851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:09.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/4245105634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: mon.a calling monitor election 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: monmap epoch 3 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: last_changed 2024-04-05T02:57:44.232912+0000 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: min_mon_release 17 (quincy) 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: election_strategy: 1 2024-04-05T03:07:09.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: mgrmap e31: y(active, since 38s), standbys: x 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: daemon grafana.a on smithi151 is in unknown state 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: mgrmap e32: y(active, since 38s), standbys: x 2024-04-05T03:07:09.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:09 smithi033 ceph-mon[97738]: from='mgr.25766 ' entity='' 2024-04-05T03:07:09.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: ignoring --setuser ceph since I am not root 2024-04-05T03:07:09.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: ignoring --setgroup ceph since I am not root 2024-04-05T03:07:09.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mgr[78492]: -- 172.21.15.151:0/1149578700 <== mon.1 v2:172.21.15.151:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56552e1d45a0 con 0x56552e1a8c00 2024-04-05T03:07:09.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:07:09.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:07:09.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: from='mgr.25766 172.21.15.33:0/2948166807' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:07:09.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3258519851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/4245105634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: mon.a calling monitor election 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: monmap epoch 3 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: last_changed 2024-04-05T02:57:44.232912+0000 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: min_mon_release 17 (quincy) 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: election_strategy: 1 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:07:09.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: mgrmap e31: y(active, since 38s), standbys: x 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: daemon grafana.a on smithi151 is in unknown state 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: mgrmap e32: y(active, since 38s), standbys: x 2024-04-05T03:07:09.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-mon[32789]: from='mgr.25766 ' entity='' 2024-04-05T03:07:10.147 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 1 2024-04-05T03:07:10.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:09 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:09.974+0000 7fa7c187c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:07:10.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:10 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:10.067+0000 7fa7c187c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:07:10.461 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:10.005+0000 7f51b3d63200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T03:07:10.461 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:10.102+0000 7f51b3d63200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T03:07:10.461 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:10 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2309438046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:10.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:10 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2309438046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:10.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:10 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2309438046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:07:10.778 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:10 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:10.701+0000 7fa7c187c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:07:10.810 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:07:11.037 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:10.776+0000 7f51b3d63200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T03:07:11.037 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:10.886+0000 7f51b3d63200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:07:11.037 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.037+0000 7f51b3d63200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:07:11.088 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:07:11.114 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:10 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:10.794+0000 7fa7c187c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T03:07:11.114 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:10 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:10.936+0000 7fa7c187c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T03:07:11.114 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:11.032+0000 7fa7c187c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:07:11.114 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:11.114+0000 7fa7c187c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:07:11.295 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.147+0000 7f51b3d63200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T03:07:11.295 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.234+0000 7f51b3d63200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T03:07:11.624 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.511+0000 7f51b3d63200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:07:11.624 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.624+0000 7f51b3d63200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:07:11.695 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:11.387+0000 7fa7c187c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T03:07:11.695 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:11.504+0000 7fa7c187c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T03:07:11.696 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:11.595+0000 7fa7c187c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:07:11.696 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:11.695+0000 7fa7c187c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:07:11.885 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.713+0000 7f51b3d63200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T03:07:11.885 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.816+0000 7f51b3d63200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T03:07:12.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:11 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:11.879+0000 7fa7c187c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:07:12.211 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:11.997+0000 7f51b3d63200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T03:07:13.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:12 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:12.876+0000 7fa7c187c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:07:13.278 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:13 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:13.007+0000 7fa7c187c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:07:13.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:13.007+0000 7f51b3d63200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T03:07:13.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:13.142+0000 7f51b3d63200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T03:07:13.555 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:13 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:13.357+0000 7fa7c187c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:07:13.555 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:13 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:13.465+0000 7fa7c187c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:07:13.555 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-05T03:07:13.555 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:13.555 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:07:13.556 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 podman[82453]: 2024-04-05 03:07:13.423911086 +0000 UTC m=+0.021182458 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:07:13.785 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:13.486+0000 7f51b3d63200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T03:07:13.786 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:13.580+0000 7f51b3d63200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T03:07:13.786 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:13.786+0000 7f51b3d63200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:07:13.855 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:13 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:13.670+0000 7fa7c187c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:07:13.855 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 podman[82453]: 2024-04-05 03:07:13.556449683 +0000 UTC m=+0.153721022 container create 150cc2082b9afa7b576138a02bf9c18dd8107f42725677f3d8c1d2d4fe16cf29 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, description=Ceph Grafana Container, name=ubi8, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, release=239.1651231664, vcs-type=git, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-05T03:07:13.855 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 podman[82453]: 2024-04-05 03:07:13.781460628 +0000 UTC m=+0.378731968 container init 150cc2082b9afa7b576138a02bf9c18dd8107f42725677f3d8c1d2d4fe16cf29 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, distribution-scope=public, com.redhat.component=ubi8-container, version=8.5, maintainer=Paul Cuzner , io.buildah.version=1.32.0, release=239.1651231664, io.openshift.expose-services=) 2024-04-05T03:07:13.856 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 podman[82453]: 2024-04-05 03:07:13.784106204 +0000 UTC m=+0.381377546 container start 150cc2082b9afa7b576138a02bf9c18dd8107f42725677f3d8c1d2d4fe16cf29 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.expose-services=, maintainer=Paul Cuzner , description=Ceph Grafana Container, vcs-type=git, vendor=Red Hat, Inc., release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, name=ubi8, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T03:07:14.177 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:13 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:13.873+0000 7fa7c187c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:07:14.177 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 bash[82453]: 150cc2082b9afa7b576138a02bf9c18dd8107f42725677f3d8c1d2d4fe16cf29 2024-04-05T03:07:14.178 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:14.178 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[82472]: logger=settings t=2024-04-05T03:07:13.939861277Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:07:14.178 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:13 smithi151 podman[82492]: 2024-04-05 03:07:13.970782087 +0000 UTC m=+0.019022121 container died 150cc2082b9afa7b576138a02bf9c18dd8107f42725677f3d8c1d2d4fe16cf29 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.expose-services=, version=8.5) 2024-04-05T03:07:14.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:13.980+0000 7f51b3d63200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T03:07:14.445 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:14 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:14.177+0000 7fa7c187c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:07:14.546 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:14.280+0000 7f51b3d63200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T03:07:14.720 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:14 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:14.445+0000 7fa7c187c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:07:14.720 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:14 smithi151 podman[82492]: 2024-04-05 03:07:14.47680535 +0000 UTC m=+0.525045381 container remove 150cc2082b9afa7b576138a02bf9c18dd8107f42725677f3d8c1d2d4fe16cf29 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , description=Ceph Grafana Container, release=239.1651231664, architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base rhel8, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-05T03:07:14.721 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:14 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:07:14.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:14.546+0000 7f51b3d63200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T03:07:14.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:14.794+0000 7f51b3d63200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:07:15.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:14 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:07:15.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:14 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:14.724+0000 7fa7c187c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T03:07:15.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:15 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:15.637+0000 7f51b3d63200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:07:16.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:15 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:15.539+0000 7fa7c187c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T03:07:16.444 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:16.118+0000 7fa7c187c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:07:16.444 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:16.219+0000 7fa7c187c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:07:16.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.229+0000 7f51b3d63200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T03:07:16.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.332+0000 7f51b3d63200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T03:07:16.702 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:16.444+0000 7fa7c187c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:07:16.702 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: [05/Apr/2024:03:07:16] ENGINE Bus STARTING 2024-04-05T03:07:16.702 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: CherryPy Checker: 2024-04-05T03:07:16.702 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: The Application mounted at '' has an empty config. 2024-04-05T03:07:16.702 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: 2024-04-05T03:07:16.702 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: [05/Apr/2024:03:07:16] ENGINE Serving on http://:::9283 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x[78488]: [05/Apr/2024:03:07:16] ENGINE Bus STARTED 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-mon[32789]: Standby manager daemon x restarted 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-mon[32789]: Standby manager daemon x started 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-mon[32789]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:07:16.703 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:16 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b[32785]: 2024-04-05T03:07:16.569+0000 7f7020113700 -1 mon.b@1(peon).osd e88 update_from_paxos full map CRC mismatch, resetting to canonical 2024-04-05T03:07:16.778 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[97738]: Standby manager daemon x restarted 2024-04-05T03:07:16.778 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[97738]: Standby manager daemon x started 2024-04-05T03:07:16.778 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[97738]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:07:16.778 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[97738]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:07:16.778 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[97738]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[97738]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[40008]: Standby manager daemon x restarted 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[40008]: Standby manager daemon x started 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-mon[40008]: from='mgr.? 172.21.15.151:0/909615960' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c[40004]: 2024-04-05T03:07:16.569+0000 7f0f5a80e700 -1 mon.c@2(peon).osd e88 update_from_paxos full map CRC mismatch, resetting to canonical 2024-04-05T03:07:16.779 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.564+0000 7f51b3d63200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:16] ENGINE Bus STARTING 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.689+0000 7f511397b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.689+0000 7f511397b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.689+0000 7f511397b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.689+0000 7f511397b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.689+0000 7f511397b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.690+0000 7f5117983700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.690+0000 7f5117983700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.690+0000 7f5117983700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.690+0000 7f5117983700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:16.780 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:16.690+0000 7f5117983700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T03:07:17.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: CherryPy Checker: 2024-04-05T03:07:17.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: The Application mounted at '' has an empty config. 2024-04-05T03:07:17.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:17.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:16] ENGINE Serving on http://:::9283 2024-04-05T03:07:17.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:07:16] ENGINE Bus STARTED 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (83s) 42s ago 5m 26.2M - 0.25.0 c8568f914cd2 a9cf7c76c519 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi151 *:3000 unknown 21s ago 4m - - 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi033.rmzwzs smithi033 running (3m) 42s ago 3m 13.9M - 17.2.7-805-g6fda41ff a49113eacc58 c7d056f2104c 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi151.cxpowy smithi151 running (3m) 21s ago 3m 15.8M - 17.2.7-805-g6fda41ff a49113eacc58 0d112d294451 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi151 *:8443,9283,8765 running (25s) 21s ago 9m 67.0M - 19.0.0-2013-g0028525c d8c35eaf3c25 090d83bbd680 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (55s) 42s ago 11m 475M - 19.0.0-2013-g0028525c d8c35eaf3c25 a6e11bc7dc6c 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (11m) 42s ago 11m 51.0M 2048M 17.2.7-805-g6fda41ff a49113eacc58 e876c4ff5784 2024-04-05T03:07:17.594 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi151 running (9m) 21s ago 9m 44.0M 2048M 17.2.7-805-g6fda41ff a49113eacc58 464b19e4a343 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (9m) 42s ago 9m 39.2M 2048M 17.2.7-805-g6fda41ff a49113eacc58 a5676722d2cb 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (80s) 42s ago 4m 8380k - 1.5.0 0da6a335fe13 6da1bdc228ef 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi151 *:9100 running (61s) 21s ago 4m 15.0M - 1.5.0 0da6a335fe13 18b78d165e69 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (8m) 42s ago 8m 48.2M 2481M 17.2.7-805-g6fda41ff a49113eacc58 d0c1cb9b20a4 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (8m) 42s ago 8m 49.7M 2481M 17.2.7-805-g6fda41ff a49113eacc58 279958cf423a 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (8m) 42s ago 8m 45.5M 2481M 17.2.7-805-g6fda41ff a49113eacc58 309a3fe45561 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (7m) 42s ago 7m 42.7M 2481M 17.2.7-805-g6fda41ff a49113eacc58 2b3b4d99eb65 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi151 running (7m) 21s ago 7m 44.7M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b7f73016d7ed 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi151 running (7m) 21s ago 7m 48.4M 2480M 17.2.7-805-g6fda41ff a49113eacc58 e2607db1cdd0 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi151 running (6m) 21s ago 6m 47.6M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2063f47e19f2 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi151 running (6m) 21s ago 6m 48.4M 2480M 17.2.7-805-g6fda41ff a49113eacc58 164624ba7a6d 2024-04-05T03:07:17.595 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi151 *:9095 running (31s) 21s ago 5m 39.1M - 2.43.0 a07b618ecd1d c6c02f0b66d8 2024-04-05T03:07:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: mgrmap e33: y(active, since 45s), standbys: x 2024-04-05T03:07:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: Active manager daemon y restarted 2024-04-05T03:07:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: Activating manager daemon y 2024-04-05T03:07:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:07:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: mgrmap e34: y(active, starting, since 0.00498383s), standbys: x 2024-04-05T03:07:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:07:17.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: Manager daemon y is now available 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:07:17.782 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:17 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:07:17.785 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:07:17.577+0000 7f5169f0e700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: mgrmap e33: y(active, since 45s), standbys: x 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: Active manager daemon y restarted 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: Activating manager daemon y 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: mgrmap e34: y(active, starting, since 0.00498383s), standbys: x 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:07:17.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:07:17.787 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: Manager daemon y is now available 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: mgrmap e33: y(active, since 45s), standbys: x 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: Active manager daemon y restarted 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: Activating manager daemon y 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: mgrmap e34: y(active, starting, since 0.00498383s), standbys: x 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.788 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi033.rmzwzs"}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi151.cxpowy"}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T03:07:17.789 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: Manager daemon y is now available 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.790 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.791 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:07:17.791 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:07:17.791 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T03:07:17.791 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.791 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: failed to encode map e88 with expected crc 2024-04-05T03:07:17.791 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:07:17.791 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:17 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T03:07:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:18 smithi151 ceph-mon[32789]: mgrmap e35: y(active, since 1.00955s), standbys: x 2024-04-05T03:07:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:18 smithi151 ceph-mon[32789]: [05/Apr/2024:03:07:17] ENGINE Bus STARTING 2024-04-05T03:07:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:18 smithi151 ceph-mon[32789]: [05/Apr/2024:03:07:17] ENGINE Serving on http://172.21.15.33:8765 2024-04-05T03:07:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:18 smithi151 ceph-mon[32789]: [05/Apr/2024:03:07:18] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:07:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:18 smithi151 ceph-mon[32789]: [05/Apr/2024:03:07:18] ENGINE Bus STARTED 2024-04-05T03:07:18.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[40008]: mgrmap e35: y(active, since 1.00955s), standbys: x 2024-04-05T03:07:18.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[40008]: [05/Apr/2024:03:07:17] ENGINE Bus STARTING 2024-04-05T03:07:18.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[40008]: [05/Apr/2024:03:07:17] ENGINE Serving on http://172.21.15.33:8765 2024-04-05T03:07:18.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[40008]: [05/Apr/2024:03:07:18] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:07:18.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[40008]: [05/Apr/2024:03:07:18] ENGINE Bus STARTED 2024-04-05T03:07:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[97738]: mgrmap e35: y(active, since 1.00955s), standbys: x 2024-04-05T03:07:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[97738]: [05/Apr/2024:03:07:17] ENGINE Bus STARTING 2024-04-05T03:07:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[97738]: [05/Apr/2024:03:07:17] ENGINE Serving on http://172.21.15.33:8765 2024-04-05T03:07:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[97738]: [05/Apr/2024:03:07:18] ENGINE Serving on https://172.21.15.33:7150 2024-04-05T03:07:18.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:18 smithi033 ceph-mon[97738]: [05/Apr/2024:03:07:18] ENGINE Bus STARTED 2024-04-05T03:07:19.009 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T03:07:19.650 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:07:19.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[97738]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T03:07:19.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[97738]: Cluster is now healthy 2024-04-05T03:07:19.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[97738]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:19.904 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[97738]: mgrmap e36: y(active, since 2s), standbys: x 2024-04-05T03:07:19.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[40008]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T03:07:19.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[40008]: Cluster is now healthy 2024-04-05T03:07:19.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[40008]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:19.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:19 smithi033 ceph-mon[40008]: mgrmap e36: y(active, since 2s), standbys: x 2024-04-05T03:07:20.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:19 smithi151 ceph-mon[32789]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T03:07:20.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:19 smithi151 ceph-mon[32789]: Cluster is now healthy 2024-04-05T03:07:20.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:19 smithi151 ceph-mon[32789]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:20.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:19 smithi151 ceph-mon[32789]: mgrmap e36: y(active, since 2s), standbys: x 2024-04-05T03:07:21.029 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:07:20] "GET /metrics HTTP/1.1" 200 31288 "" "Prometheus/2.43.0" 2024-04-05T03:07:21.608 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[97738]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:21.608 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[97738]: mgrmap e37: y(active, since 4s), standbys: x 2024-04-05T03:07:21.608 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:21.608 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:21.608 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[40008]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:21.608 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[40008]: mgrmap e37: y(active, since 4s), standbys: x 2024-04-05T03:07:21.608 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:21.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:21 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:21.623 INFO:teuthology.orchestra.run.smithi033.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:07:21.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:21 smithi151 ceph-mon[32789]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:21.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:21 smithi151 ceph-mon[32789]: mgrmap e37: y(active, since 4s), standbys: x 2024-04-05T03:07:21.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:21 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:21.840 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:21 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.348 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'sleep 60' 2024-04-05T03:07:22.676 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:22 smithi151 ceph-mon[32789]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:07:22.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:22 smithi151 ceph-mon[32789]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:07:22.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:22 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:22 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.677 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:22 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[40008]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[40008]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[97738]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[97738]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:22 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:23.140 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:07:23.898 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:23 smithi033 ceph-mon[40008]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-05T03:07:23.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:23 smithi033 ceph-mon[97738]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-05T03:07:24.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:23 smithi151 ceph-mon[32789]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-05T03:07:24.891 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:24 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-05T03:07:24.891 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:24 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:24.891 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:24 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:07:25.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:24 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:25.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:24 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:24 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:25.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:24 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:24 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:24 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:25.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:25 smithi151 podman[86266]: 2024-04-05 03:07:25.447655877 +0000 UTC m=+0.018365037 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:07:25.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:25 smithi151 podman[86266]: 2024-04-05 03:07:25.598734977 +0000 UTC m=+0.169444179 container create a4a573713303920237591b9031a17ffb6e79e71b4ebda220a3ab1c46ec8ac7a1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-05T03:07:25.842 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:25 smithi151 podman[86266]: 2024-04-05 03:07:25.843164727 +0000 UTC m=+0.413873890 container init a4a573713303920237591b9031a17ffb6e79e71b4ebda220a3ab1c46ec8ac7a1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, vcs-type=git, io.buildah.version=1.32.0) 2024-04-05T03:07:26.093 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:25 smithi151 podman[86266]: 2024-04-05 03:07:25.846116501 +0000 UTC m=+0.416825662 container start a4a573713303920237591b9031a17ffb6e79e71b4ebda220a3ab1c46ec8ac7a1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, name=ubi8, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, version=8.5) 2024-04-05T03:07:26.093 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:25 smithi151 bash[86266]: a4a573713303920237591b9031a17ffb6e79e71b4ebda220a3ab1c46ec8ac7a1 2024-04-05T03:07:26.093 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:25 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:26.093 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[86529]: logger=settings t=2024-04-05T03:07:26.004515179Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:07:26.093 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:26 smithi151 podman[86765]: 2024-04-05 03:07:26.030668968 +0000 UTC m=+0.019063268 container died a4a573713303920237591b9031a17ffb6e79e71b4ebda220a3ab1c46ec8ac7a1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.buildah.version=1.32.0, name=ubi8, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, architecture=x86_64, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, release=239.1651231664) 2024-04-05T03:07:26.388 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:07:26.388 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:26.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:26.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:07:26.389 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:26 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:07:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:07:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:26.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:26.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:26.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: Updating smithi033:/etc/ceph/ceph.conf 2024-04-05T03:07:26.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:26 smithi033 ceph-mon[97738]: Updating smithi151:/etc/ceph/ceph.conf 2024-04-05T03:07:26.928 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:26 smithi151 podman[86765]: 2024-04-05 03:07:26.664927395 +0000 UTC m=+0.653321675 container remove a4a573713303920237591b9031a17ffb6e79e71b4ebda220a3ab1c46ec8ac7a1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, name=ubi8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, architecture=x86_64, description=Ceph Grafana Container, version=8.5) 2024-04-05T03:07:26.928 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:26 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:07:27.211 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:26 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:07:27.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:07:27.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:27.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:27.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:27 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: Updating smithi151:/etc/ceph/ceph.client.admin.keyring 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: Updating smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: Updating smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:27.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:27.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:27.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:27.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:27 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:07:28.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:28 smithi033 ceph-mon[40008]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:07:28.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:28 smithi033 ceph-mon[40008]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:28.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:28 smithi033 ceph-mon[40008]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:07:28.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:28 smithi033 ceph-mon[97738]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:07:28.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:28 smithi033 ceph-mon[97738]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:28.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:28 smithi033 ceph-mon[97738]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:07:28.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:28 smithi151 ceph-mon[32789]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-05T03:07:28.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:28 smithi151 ceph-mon[32789]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:28.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:28 smithi151 ceph-mon[32789]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T03:07:29.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:07:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:07:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:07:30.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:30 smithi033 ceph-mon[97738]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:30.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:30 smithi033 ceph-mon[40008]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:30 smithi151 ceph-mon[32789]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-05T03:07:31.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:07:30] "GET /metrics HTTP/1.1" 200 31288 "" "Prometheus/2.43.0" 2024-04-05T03:07:32.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:32 smithi033 ceph-mon[40008]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:32.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:32 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:32.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:32 smithi033 ceph-mon[97738]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:32.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:32 smithi151 ceph-mon[32789]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:32 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:34.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:34 smithi033 ceph-mon[97738]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:34.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:34 smithi033 ceph-mon[40008]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:34.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:34 smithi151 ceph-mon[32789]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:36.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:36 smithi033 ceph-mon[97738]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:36.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:36 smithi033 ceph-mon[40008]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:36 smithi151 ceph-mon[32789]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:37.362 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-05T03:07:37.362 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:37.362 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:07:37.362 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 podman[87305]: 2024-04-05 03:07:37.248652837 +0000 UTC m=+0.097076965 container create 7e448f0d2b8a31d2d10d9c02c2dc14f17dfa7392ed9651c12c66eb8fb403ab49 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, architecture=x86_64, name=ubi8, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, release=239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vcs-type=git, distribution-scope=public) 2024-04-05T03:07:37.362 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 podman[87305]: 2024-04-05 03:07:37.169576385 +0000 UTC m=+0.018000513 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:07:37.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 podman[87305]: 2024-04-05 03:07:37.484037993 +0000 UTC m=+0.332462123 container init 7e448f0d2b8a31d2d10d9c02c2dc14f17dfa7392ed9651c12c66eb8fb403ab49 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, vcs-type=git, io.openshift.expose-services=, description=Ceph Grafana Container, vendor=Red Hat, Inc., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, release=239.1651231664) 2024-04-05T03:07:37.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 podman[87305]: 2024-04-05 03:07:37.486601445 +0000 UTC m=+0.335025575 container start 7e448f0d2b8a31d2d10d9c02c2dc14f17dfa7392ed9651c12c66eb8fb403ab49 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , release=239.1651231664, build-date=2022-04-29T11:48:11.055897, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, name=ubi8, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-05T03:07:37.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 bash[87305]: 7e448f0d2b8a31d2d10d9c02c2dc14f17dfa7392ed9651c12c66eb8fb403ab49 2024-04-05T03:07:37.778 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:37.779 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[87320]: logger=settings t=2024-04-05T03:07:37.639600062Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:07:37.779 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:37 smithi151 podman[87337]: 2024-04-05 03:07:37.665680763 +0000 UTC m=+0.018575876 container died 7e448f0d2b8a31d2d10d9c02c2dc14f17dfa7392ed9651c12c66eb8fb403ab49 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, description=Ceph Grafana Container, version=8.5, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, io.openshift.tags=base rhel8) 2024-04-05T03:07:38.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:38 smithi151 podman[87337]: 2024-04-05 03:07:38.196114515 +0000 UTC m=+0.549009629 container remove 7e448f0d2b8a31d2d10d9c02c2dc14f17dfa7392ed9651c12c66eb8fb403ab49 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, release=239.1651231664, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, description=Ceph Grafana Container, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vcs-type=git, name=ubi8, build-date=2022-04-29T11:48:11.055897, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-05T03:07:38.209 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:38 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:07:38.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:38 smithi033 ceph-mon[97738]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:38.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:38 smithi033 ceph-mon[40008]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:38.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:38 smithi151 ceph-mon[32789]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:38.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:38 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:07:39.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:07:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:07:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:07:40.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:40 smithi033 ceph-mon[97738]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:40.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:40 smithi033 ceph-mon[40008]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:40.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:40 smithi151 ceph-mon[32789]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:07:40] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-04-05T03:07:42.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:42 smithi151 ceph-mon[32789]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:42 smithi033 ceph-mon[97738]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:42 smithi033 ceph-mon[40008]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:44.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:44 smithi151 ceph-mon[32789]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:44.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:44 smithi033 ceph-mon[40008]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:44.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:44 smithi033 ceph-mon[97738]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:46.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:46 smithi151 ceph-mon[32789]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:46.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:46 smithi033 ceph-mon[97738]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:46.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:46 smithi033 ceph-mon[40008]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:47.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:47 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:47 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:48 smithi151 ceph-mon[32789]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:48 smithi033 ceph-mon[97738]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:48.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:48 smithi033 ceph-mon[40008]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:48.896 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:07:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:07:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:07:48.896 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:48 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-05T03:07:48.896 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:48 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:48.896 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:48 smithi151 systemd[1]: Starting Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:07:48.896 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:48 smithi151 podman[87442]: 2024-04-05 03:07:48.67397817 +0000 UTC m=+0.018387553 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T03:07:48.896 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:48 smithi151 podman[87442]: 2024-04-05 03:07:48.780001985 +0000 UTC m=+0.124411353 container create c4a5aef31eb29043f83a7adf6236652c81a14e91b5beb6bad7c9b7be49297511 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , name=ubi8, io.openshift.expose-services=, version=8.5, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public) 2024-04-05T03:07:49.153 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:48 smithi151 podman[87442]: 2024-04-05 03:07:48.999518687 +0000 UTC m=+0.343928066 container init c4a5aef31eb29043f83a7adf6236652c81a14e91b5beb6bad7c9b7be49297511 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.buildah.version=1.32.0, maintainer=Paul Cuzner , io.openshift.expose-services=, vcs-type=git, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public) 2024-04-05T03:07:49.153 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 podman[87442]: 2024-04-05 03:07:49.00226428 +0000 UTC m=+0.346673659 container start c4a5aef31eb29043f83a7adf6236652c81a14e91b5beb6bad7c9b7be49297511 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5) 2024-04-05T03:07:49.153 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 bash[87442]: c4a5aef31eb29043f83a7adf6236652c81a14e91b5beb6bad7c9b7be49297511 2024-04-05T03:07:49.153 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 systemd[1]: Started Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:07:49.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a[87456]: logger=settings t=2024-04-05T03:07:49.154178598Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T03:07:49.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 podman[87473]: 2024-04-05 03:07:49.183343141 +0000 UTC m=+0.018553937 container died c4a5aef31eb29043f83a7adf6236652c81a14e91b5beb6bad7c9b7be49297511 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , release=239.1651231664) 2024-04-05T03:07:50.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 podman[87473]: 2024-04-05 03:07:49.754472173 +0000 UTC m=+0.589682959 container remove c4a5aef31eb29043f83a7adf6236652c81a14e91b5beb6bad7c9b7be49297511 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-grafana-a, distribution-scope=public, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, version=8.5, architecture=x86_64, com.redhat.component=ubi8-container, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=) 2024-04-05T03:07:50.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T03:07:50.028 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:07:49 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:07:50.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:50 smithi151 ceph-mon[32789]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:50.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:50 smithi033 ceph-mon[97738]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:50.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:50 smithi033 ceph-mon[40008]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:51.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:07:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:07:50] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2024-04-05T03:07:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:52 smithi151 ceph-mon[32789]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:52.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:52 smithi033 ceph-mon[97738]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:52.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:52 smithi033 ceph-mon[40008]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:54.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:54 smithi151 ceph-mon[32789]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:54.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:54 smithi033 ceph-mon[97738]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:54.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:54 smithi033 ceph-mon[40008]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:56 smithi151 ceph-mon[32789]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:56 smithi033 ceph-mon[97738]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:56.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:56 smithi033 ceph-mon[40008]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:58.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:07:58 smithi151 ceph-mon[32789]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:58.532 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:07:58 smithi033 ceph-mon[97738]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:58.533 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:07:58 smithi033 ceph-mon[40008]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:07:59.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:07:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:07:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:08:00.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:00 smithi151 ceph-mon[32789]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:00.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:08:00 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-04-05T03:08:00.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:08:00 smithi151 systemd[1]: Stopped Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:08:00.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:08:00 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Start request repeated too quickly. 2024-04-05T03:08:00.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:08:00 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T03:08:00.528 INFO:journalctl@ceph.grafana.a.smithi151.stdout:Apr 05 03:08:00 smithi151 systemd[1]: Failed to start Ceph grafana.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:08:00.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:00 smithi033 ceph-mon[97738]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:00.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:00 smithi033 ceph-mon[40008]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:01.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:08:00] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2024-04-05T03:08:02.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:02 smithi151 ceph-mon[32789]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:02.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:02 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:02.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:02 smithi033 ceph-mon[97738]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:02.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:02.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:02 smithi033 ceph-mon[40008]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:02.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:02 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:04.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:04 smithi151 ceph-mon[32789]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:04.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:04 smithi033 ceph-mon[97738]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:04.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:04 smithi033 ceph-mon[40008]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:06.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:06 smithi151 ceph-mon[32789]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:06.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:06 smithi033 ceph-mon[97738]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:06.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:06 smithi033 ceph-mon[40008]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:08.658 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:08 smithi151 ceph-mon[32789]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:08.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:08 smithi033 ceph-mon[97738]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:08.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:08 smithi033 ceph-mon[40008]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:09.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:08:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:08:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:08:10.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:10 smithi033 ceph-mon[97738]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:10.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:10 smithi033 ceph-mon[40008]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:10.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:10 smithi151 ceph-mon[32789]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:11.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:08:10] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.43.0" 2024-04-05T03:08:12.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:12 smithi033 ceph-mon[40008]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:12.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:12 smithi033 ceph-mon[97738]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:12 smithi151 ceph-mon[32789]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:14.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:14 smithi033 ceph-mon[40008]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:14.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:14 smithi033 ceph-mon[97738]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:14 smithi151 ceph-mon[32789]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:16.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:16 smithi033 ceph-mon[97738]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:16.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:16 smithi033 ceph-mon[40008]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:16.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:16 smithi151 ceph-mon[32789]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:17.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:17.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:17 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:17 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:18 smithi033 ceph-mon[97738]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:18.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:18 smithi033 ceph-mon[40008]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:18 smithi151 ceph-mon[32789]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:18.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:08:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:08:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:08:20.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:20 smithi033 ceph-mon[97738]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:20.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:20 smithi033 ceph-mon[40008]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:20 smithi151 ceph-mon[32789]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:21.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:08:20] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-04-05T03:08:22.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:22 smithi033 ceph-mon[97738]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:22 smithi033 ceph-mon[40008]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:22 smithi151 ceph-mon[32789]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:24 smithi033 ceph-mon[97738]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:24.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:24 smithi033 ceph-mon[40008]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:24 smithi151 ceph-mon[32789]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:25.807 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade resume' 2024-04-05T03:08:26.078 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:08:26.452 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:26 smithi033 ceph-mon[97738]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:26.453 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:26 smithi033 ceph-mon[40008]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:26 smithi151 ceph-mon[32789]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:27.600 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:27 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:27.600 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:27 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:28.070 INFO:teuthology.orchestra.run.smithi033.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:28.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:28 smithi033 ceph-mon[97738]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:28.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:28 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:28.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:28 smithi033 ceph-mon[40008]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:28.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:28 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:28.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:28 smithi151 ceph-mon[32789]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:28.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:28 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:28.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:08:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:08:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:08:28.901 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-05T03:08:29.292 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:08:29.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:29 smithi033 ceph-mon[40008]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:29.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:29 smithi033 ceph-mon[40008]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:29.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:29 smithi033 ceph-mon[97738]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:29.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:29 smithi033 ceph-mon[97738]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:29.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:29 smithi151 ceph-mon[32789]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:29.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:29 smithi151 ceph-mon[32789]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:30.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:30 smithi151 ceph-mon[32789]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:30 smithi033 ceph-mon[97738]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:30 smithi033 ceph-mon[40008]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:31.097 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:08:30] "GET /metrics HTTP/1.1" 200 33978 "" "Prometheus/2.43.0" 2024-04-05T03:08:31.427 INFO:teuthology.orchestra.run.smithi033.stdout:wait for majority of mons to upgrade 2024-04-05T03:08:32.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:31 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:32.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:31 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:32.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:31 smithi151 ceph-mon[32789]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:32.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:31 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:32.065 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:32.065 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:32.065 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[40008]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:32.065 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:32.065 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:32.065 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:32.066 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[97738]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:32.066 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:31 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:32.090 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-05T03:08:32.810 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:08:34.272 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:34 smithi033 ceph-mon[97738]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:34.272 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:34 smithi033 ceph-mon[40008]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:34.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:34 smithi151 ceph-mon[32789]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:35.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:35.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:08:35.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:35 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:08:35.574 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:08:35.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:35 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-05T03:08:36.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[97738]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:36.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[97738]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:36.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[97738]: Upgrade: It appears safe to stop mon.c 2024-04-05T03:08:36.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1270535763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[40008]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[40008]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[40008]: Upgrade: It appears safe to stop mon.c 2024-04-05T03:08:36.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:36 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/1270535763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:36 smithi151 ceph-mon[32789]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:36 smithi151 ceph-mon[32789]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:36 smithi151 ceph-mon[32789]: Upgrade: It appears safe to stop mon.c 2024-04-05T03:08:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:36 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/1270535763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[97738]: Upgrade: Updating mon.c 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[97738]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[97738]: Deploying daemon mon.c on smithi033 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[97738]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[40008]: Upgrade: Updating mon.c 2024-04-05T03:08:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[40008]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:37.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:08:37.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:08:37.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[40008]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:37.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[40008]: Deploying daemon mon.c on smithi033 2024-04-05T03:08:37.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:37 smithi033 ceph-mon[40008]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:38.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:37 smithi151 ceph-mon[32789]: Upgrade: Updating mon.c 2024-04-05T03:08:38.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:37 smithi151 ceph-mon[32789]: from='mgr.25934 ' entity='mgr.y' 2024-04-05T03:08:38.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:37 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:08:38.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:37 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:08:38.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:37 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:38.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:37 smithi151 ceph-mon[32789]: Deploying daemon mon.c on smithi033 2024-04-05T03:08:38.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:37 smithi151 ceph-mon[32789]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:38.803 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2951339734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:38.803 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:38 smithi033 systemd[1]: Stopping Ceph mon.c for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:08:38.803 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-mon[40008]: from='client.? 172.21.15.33:0/2951339734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:39.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:38 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/2951339734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:39.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:08:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:08:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c[40004]: 2024-04-05T03:08:38.807+0000 7f0f60019700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c[40004]: 2024-04-05T03:08:38.807+0000 7f0f60019700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:38 smithi033 podman[101582]: 2024-04-05 03:08:38.827527476 +0000 UTC m=+0.132238729 container died a5676722d2cbcfc8f3dcc70f77f224dd3071e60acf8a9229c0f06013ccf2d26a (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:08:38] ENGINE Bus STOPPING 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:08:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:08:38] ENGINE Bus STOPPED 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:08:38] ENGINE Bus STARTING 2024-04-05T03:08:39.110 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:08:38] ENGINE Serving on http://:::9283 2024-04-05T03:08:39.111 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:08:38] ENGINE Bus STARTED 2024-04-05T03:08:39.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:39 smithi033 podman[101582]: 2024-04-05 03:08:39.324018049 +0000 UTC m=+0.628729335 container cleanup a5676722d2cbcfc8f3dcc70f77f224dd3071e60acf8a9229c0f06013ccf2d26a (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-6fda41f, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-05T03:08:39.434 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:39 smithi033 bash[101582]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c 2024-04-05T03:08:39.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:39 smithi033 podman[101596]: 2024-04-05 03:08:39.59251533 +0000 UTC m=+0.767681856 container remove a5676722d2cbcfc8f3dcc70f77f224dd3071e60acf8a9229c0f06013ccf2d26a (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, io.buildah.version=1.33.5) 2024-04-05T03:08:40.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:39 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.c.service: Deactivated successfully. 2024-04-05T03:08:40.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:39 smithi033 systemd[1]: Stopped Ceph mon.c for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:08:40.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:39 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.c.service: Consumed 7.573s CPU time. 2024-04-05T03:08:40.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:39 smithi033 systemd[1]: Starting Ceph mon.c for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:08:40.439 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 podman[101761]: 2024-04-05 03:08:40.08911676 +0000 UTC m=+0.018332889 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:40.439 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 podman[101761]: 2024-04-05 03:08:40.19376525 +0000 UTC m=+0.122981364 container create fb77798ae736f6b2fd218b802ece033ee10c821766fb2f3fc5b6f901e092f69c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.schema-version=1.0) 2024-04-05T03:08:40.439 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 podman[101761]: 2024-04-05 03:08:40.413976224 +0000 UTC m=+0.343192332 container init fb77798ae736f6b2fd218b802ece033ee10c821766fb2f3fc5b6f901e092f69c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-05T03:08:40.439 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 podman[101761]: 2024-04-05 03:08:40.41748554 +0000 UTC m=+0.346701654 container start fb77798ae736f6b2fd218b802ece033ee10c821766fb2f3fc5b6f901e092f69c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, ceph=True, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev), process ceph-mon, pid 2 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: pidfile_write: ignore empty --pid-file 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: load: jerasure load: lrc 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: RocksDB version: 7.9.2 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Git sha 0 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Compile date 2024-04-04 02:01:22 2024-04-05T03:08:40.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: DB SUMMARY 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: DB Session ID: 198XI2D935PKKHRS6D7Y 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: CURRENT file: CURRENT 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 1427662 ; 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.error_if_exists: 0 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.create_if_missing: 0 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.paranoid_checks: 1 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T03:08:40.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.env: 0x55e30b2e2c60 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.fs: PosixFileSystem 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.info_log: 0x55e30cbafae0 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.statistics: (nil) 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.use_fsync: 0 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_log_file_size: 0 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.allow_fallocate: 1 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T03:08:40.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.use_direct_reads: 0 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.db_log_dir: 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.wal_dir: 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.write_buffer_manager: 0x55e30cb82820 2024-04-05T03:08:40.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.unordered_write: 0 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.row_cache: None 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.wal_filter: None 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T03:08:40.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.two_write_queues: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.wal_compression: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.atomic_flush: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.log_readahead_size: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T03:08:40.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_background_jobs: 2 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_background_compactions: -1 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_subcompactions: 1 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T03:08:40.738 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_open_files: -1 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_background_flushes: -1 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Compression algorithms supported: 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kZSTD supported: 0 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kXpressCompression supported: 0 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kZlibCompression supported: 1 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kSnappyCompression supported: 1 2024-04-05T03:08:40.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kLZ4Compression supported: 1 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: kBZip2Compression supported: 0 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.merge_operator: 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_filter: None 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e30cbafbe0) 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-04-05T03:08:40.740 INFO:journalctl@ceph.mon.c.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_type: 0 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_block_index_type: 0 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_shortening: 1 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: checksum: 4 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: no_block_cache: 0 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache: 0x55e30cbb3090 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_options: 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: capacity : 536870912 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: num_shard_bits : 4 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: strict_capacity_limit : 0 2024-04-05T03:08:40.741 INFO:journalctl@ceph.mon.c.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_compressed: (nil) 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: persistent_cache: (nil) 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_size: 4096 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_size_deviation: 10 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_restart_interval: 16 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_block_restart_interval: 1 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: metadata_block_size: 4096 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: partition_filters: 0 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: use_delta_encoding: 1 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: filter_policy: bloomfilter 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: whole_key_filtering: 1 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: verify_compression: 0 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-04-05T03:08:40.742 INFO:journalctl@ceph.mon.c.smithi033.stdout: format_version: 5 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout: enable_index_compression: 1 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_align: 0 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_auto_readahead_size: 262144 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout: prepopulate_block_cache: 0 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout: initial_auto_readahead_size: 8192 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout: num_file_reads_for_auto_readahead: 2 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression: NoCompression 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.num_levels: 7 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T03:08:40.743 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T03:08:40.744 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T03:08:40.745 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.arena_block_size: 1048576 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T03:08:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.table_properties_collectors: 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.inplace_update_support: 0 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.bloom_locality: 0 2024-04-05T03:08:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.max_successive_merges: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.ttl: 2592000 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.enable_blob_files: false 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.min_blob_size: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-05T03:08:40.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.blob_file_starting_level: 0 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12962, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 80b13a6a-90cf-4133-a9a9-416fc0bc0568 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286520446541, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286520450899, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 861777, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12967, "largest_seqno": 13883, "table_properties": {"data_size": 857318, "index_size": 2549, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 7260, "raw_average_key_size": 21, "raw_value_size": 849844, "raw_average_value_size": 2552, "num_data_blocks": 116, "num_entries": 333, "num_filter_entries": 333, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712286520, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "80b13a6a-90cf-4133-a9a9-416fc0bc0568", "db_session_id": "198XI2D935PKKHRS6D7Y", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286520450992, "job": 1, "event": "recovery_finished"} 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T03:08:40.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e30cca4000 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: DB pointer 0x55e30cc8e000 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** DB Stats ** 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T03:08:40.750 INFO:journalctl@ceph.mon.c.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: L0 1/0 841.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 382.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: L6 1/0 8.31 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: Sum 2/0 9.13 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 382.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 382.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 382.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-05T03:08:40.751 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative compaction: 0.00 GB write, 126.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval compaction: 0.00 GB write, 126.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: Block cache BinnedLRUCache@0x55e30cbb3090#2 capacity: 512.00 MB usage: 3.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.58 KB,0.000491738%) Misc(1,0.00 KB,0%) 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: starting mon.c rank 2 at public addrs [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] at bind addrs [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???) e3 preinit fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???).mds e8 new map 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???).mds e8 print_map 2024-04-05T03:08:40.752 INFO:journalctl@ceph.mon.c.smithi033.stdout: e8 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: legacy client fscid: 1 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: Filesystem 'foo' (1) 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: fs_name foo 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: epoch 7 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: created 2024-04-05T03:02:48.450117+0000 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: modified 2024-04-05T03:03:33.860260+0000 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: tableserver 0 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: root 0 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: session_timeout 60 2024-04-05T03:08:40.753 INFO:journalctl@ceph.mon.c.smithi033.stdout: session_autoclose 300 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_file_size 1099511627776 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_xattr_size 65536 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: required_client_features {} 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: last_failure 0 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: last_failure_osd_epoch 0 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_mds 1 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: in 0 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: up {0=14718} 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: failed 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: damaged 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: stopped 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_pools [4] 2024-04-05T03:08:40.754 INFO:journalctl@ceph.mon.c.smithi033.stdout: metadata_pool 3 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: inline_data disabled 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: balancer 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: bal_rank_mask -1 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: standby_count_wanted 1 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: qdb_cluster leader: 0 members: 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: [mds.foo.smithi033.rmzwzs{0:14718} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: Standby daemons: 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout: [mds.foo.smithi151.cxpowy{-1:24535} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???).osd e88 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:08:40.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:08:40.756 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-mon[101783]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-05T03:08:40.756 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 bash[101761]: fb77798ae736f6b2fd218b802ece033ee10c821766fb2f3fc5b6f901e092f69c 2024-04-05T03:08:40.756 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:40 smithi033 systemd[1]: Started Ceph mon.c for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:08:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:08:40] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2024-04-05T03:08:47.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: mon.c calling monitor election 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: mon.b calling monitor election 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: mon.a calling monitor election 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: monmap epoch 3 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: last_changed 2024-04-05T02:57:44.232912+0000 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: min_mon_release 17 (quincy) 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: election_strategy: 1 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:08:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: mgrmap e37: y(active, since 90s), standbys: x 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[101783]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: mon.c calling monitor election 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: mon.b calling monitor election 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: mon.a calling monitor election 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: monmap epoch 3 2024-04-05T03:08:47.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: last_changed 2024-04-05T02:57:44.232912+0000 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: min_mon_release 17 (quincy) 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: election_strategy: 1 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: mgrmap e37: y(active, since 90s), standbys: x 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:08:47.983 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:47 smithi033 ceph-mon[97738]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: mon.c calling monitor election 2024-04-05T03:08:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: mon.b calling monitor election 2024-04-05T03:08:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: mon.a calling monitor election 2024-04-05T03:08:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: monmap epoch 3 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: last_changed 2024-04-05T02:57:44.232912+0000 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: min_mon_release 17 (quincy) 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: election_strategy: 1 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: mgrmap e37: y(active, since 90s), standbys: x 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:08:48.029 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:08:48.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:47 smithi151 ceph-mon[32789]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:08:48.793 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 2 2024-04-05T03:08:49.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:08:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:08:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:08:49.684 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:08:49.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:49 smithi033 ceph-mon[101783]: mgrmap e38: y(active, since 92s), standbys: x 2024-04-05T03:08:49.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:49 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3856887772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:49.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:49 smithi033 ceph-mon[101783]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:49.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:49 smithi033 ceph-mon[97738]: mgrmap e38: y(active, since 92s), standbys: x 2024-04-05T03:08:49.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:49 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3856887772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:49.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:49 smithi033 ceph-mon[97738]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:50.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:49 smithi151 ceph-mon[32789]: mgrmap e38: y(active, since 92s), standbys: x 2024-04-05T03:08:50.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:49 smithi151 ceph-mon[32789]: from='client.? 172.21.15.33:0/3856887772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:08:50.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:49 smithi151 ceph-mon[32789]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:50.059 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:08:50.460 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:08:50.458+0000 7f5169f0e700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-05T03:08:51.073 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:08:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:08:50] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.43.0" 2024-04-05T03:08:51.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:51 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:51.372 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:51 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:51.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:51 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:51.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:51 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:51 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:51 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (2m) 1s ago 6m 28.4M - 0.25.0 c8568f914cd2 a9cf7c76c519 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi151 *:3000 error 21s ago 6m - - 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi033.rmzwzs smithi033 running (5m) 1s ago 5m 15.6M - 17.2.7-805-g6fda41ff a49113eacc58 c7d056f2104c 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi151.cxpowy smithi151 running (5m) 21s ago 5m 16.2M - 17.2.7-805-g6fda41ff a49113eacc58 0d112d294451 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi151 *:8443,9283,8765 running (119s) 21s ago 10m 449M - 19.0.0-2013-g0028525c d8c35eaf3c25 090d83bbd680 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (2m) 1s ago 12m 499M - 19.0.0-2013-g0028525c d8c35eaf3c25 a6e11bc7dc6c 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (105s) 1s ago 12m 39.8M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 7657e117c64f 2024-04-05T03:08:51.831 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi151 running (11m) 21s ago 11m 49.4M 2048M 17.2.7-805-g6fda41ff a49113eacc58 464b19e4a343 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (11s) 1s ago 11m 17.4M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 fb77798ae736 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (2m) 1s ago 6m 15.3M - 1.5.0 0da6a335fe13 6da1bdc228ef 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi151 *:9100 running (2m) 21s ago 6m 16.5M - 1.5.0 0da6a335fe13 18b78d165e69 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (10m) 1s ago 10m 49.2M 2481M 17.2.7-805-g6fda41ff a49113eacc58 d0c1cb9b20a4 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (10m) 1s ago 10m 50.9M 2481M 17.2.7-805-g6fda41ff a49113eacc58 279958cf423a 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (9m) 1s ago 9m 46.4M 2481M 17.2.7-805-g6fda41ff a49113eacc58 309a3fe45561 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (9m) 1s ago 9m 43.8M 2481M 17.2.7-805-g6fda41ff a49113eacc58 2b3b4d99eb65 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi151 running (8m) 21s ago 8m 44.7M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b7f73016d7ed 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi151 running (8m) 21s ago 8m 49.0M 2480M 17.2.7-805-g6fda41ff a49113eacc58 e2607db1cdd0 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi151 running (8m) 21s ago 8m 48.2M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2063f47e19f2 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi151 running (7m) 21s ago 7m 49.2M 2480M 17.2.7-805-g6fda41ff a49113eacc58 164624ba7a6d 2024-04-05T03:08:51.832 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi151 *:9095 running (2m) 21s ago 7m 61.2M - 2.43.0 a07b618ecd1d c6c02f0b66d8 2024-04-05T03:08:52.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:52 smithi033 ceph-mon[97738]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:52.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:52 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:52.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:52 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:52.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:52 smithi033 ceph-mon[101783]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:52.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:52 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:52.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:52 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:52.535 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T03:08:52.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:52 smithi151 ceph-mon[32789]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:52.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:52 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:52.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:52 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:53.455 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:08:53.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:53 smithi033 ceph-mon[97738]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:53.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:53 smithi033 ceph-mon[101783]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:53.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:53 smithi151 ceph-mon[32789]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:54.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:54 smithi033 ceph-mon[97738]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:54.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:54 smithi033 ceph-mon[101783]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:54.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:54 smithi151 ceph-mon[32789]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:56.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:56 smithi033 ceph-mon[101783]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:56.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:56 smithi033 ceph-mon[97738]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:56 smithi151 ceph-mon[32789]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-05T03:08:56.996 INFO:teuthology.orchestra.run.smithi033.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:57.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 263 B/s wr, 0 op/s 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: Upgrade: It appears safe to stop mon.b 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:57.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 263 B/s wr, 0 op/s 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: Upgrade: It appears safe to stop mon.b 2024-04-05T03:08:57.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:57.684 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:57.684 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.709 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'sleep 60' 2024-04-05T03:08:57.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:57.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:57.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 263 B/s wr, 0 op/s 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: Upgrade: It appears safe to stop mon.b 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:57.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:57 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:57.975 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:08:58.957 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:08:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:08:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:08:59.384 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[101783]: Upgrade: Updating mon.b 2024-04-05T03:08:59.384 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:59.384 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:08:59.384 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:08:59.384 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:59.384 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[101783]: Deploying daemon mon.b on smithi151 2024-04-05T03:08:59.385 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[97738]: Upgrade: Updating mon.b 2024-04-05T03:08:59.385 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:59.385 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:08:59.385 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:08:59.385 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:59.385 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:08:59 smithi033 ceph-mon[97738]: Deploying daemon mon.b on smithi151 2024-04-05T03:08:59.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:59 smithi151 ceph-mon[32789]: Upgrade: Updating mon.b 2024-04-05T03:08:59.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:59 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:08:59.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:59 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:08:59.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:59 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:08:59.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:59 smithi151 ceph-mon[32789]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:59.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:08:59 smithi151 ceph-mon[32789]: Deploying daemon mon.b on smithi151 2024-04-05T03:09:00.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:00 smithi151 systemd[1]: Stopping Ceph mon.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:09:00.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:00 smithi151 ceph-mon[32789]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:00.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b[32785]: 2024-04-05T03:09:00.332+0000 7f702591e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:09:00.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b[32785]: 2024-04-05T03:09:00.332+0000 7f702591e700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-05T03:09:00.529 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:00.087Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", public_addr=\"172.21.15.33\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.33\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-05T03:09:00.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:00 smithi033 ceph-mon[101783]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:00.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:00 smithi033 ceph-mon[97738]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:00.807 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:00 smithi151 podman[88509]: 2024-04-05 03:09:00.537846859 +0000 UTC m=+0.301006771 container died 464b19e4a343054bcb795ba56e4d267627635e640a8d7d198471cabf5a58264c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=quincy-6fda41f, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-05T03:09:01.008 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:09:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:09:00] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.43.0" 2024-04-05T03:09:01.191 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:00 smithi151 podman[88509]: 2024-04-05 03:09:00.916314616 +0000 UTC m=+0.679474526 container cleanup 464b19e4a343054bcb795ba56e4d267627635e640a8d7d198471cabf5a58264c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-05T03:09:01.191 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:00 smithi151 bash[88509]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b 2024-04-05T03:09:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 podman[88524]: 2024-04-05 03:09:01.192672943 +0000 UTC m=+0.651840604 container remove 464b19e4a343054bcb795ba56e4d267627635e640a8d7d198471cabf5a58264c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-05T03:09:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.b.service: Deactivated successfully. 2024-04-05T03:09:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 systemd[1]: Stopped Ceph mon.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:09:01.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.b.service: Consumed 9.076s CPU time. 2024-04-05T03:09:01.857 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 systemd[1]: Starting Ceph mon.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:09:01.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 podman[88653]: 2024-04-05 03:09:01.654428715 +0000 UTC m=+0.017903688 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:01.858 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 podman[88653]: 2024-04-05 03:09:01.761001665 +0000 UTC m=+0.124476634 container create 634be946ebb9a25aefab251da1f74329c86b43354fed1500f82fe92f4ead4737 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-05T03:09:01.995 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 podman[88653]: 2024-04-05 03:09:01.963763889 +0000 UTC m=+0.327238859 container init 634be946ebb9a25aefab251da1f74329c86b43354fed1500f82fe92f4ead4737 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:09:01.995 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 podman[88653]: 2024-04-05 03:09:01.966912684 +0000 UTC m=+0.330387655 container start 634be946ebb9a25aefab251da1f74329c86b43354fed1500f82fe92f4ead4737 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.license=GPLv2, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev), process ceph-mon, pid 2 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: pidfile_write: ignore empty --pid-file 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: load: jerasure load: lrc 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: RocksDB version: 7.9.2 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Git sha 0 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Compile date 2024-04-04 02:01:22 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: DB SUMMARY 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: DB Session ID: JJTU19BWO8I9N4GACQGH 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: CURRENT file: CURRENT 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2024-04-05T03:09:01.996 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-05T03:09:01.997 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 2256694 ; 2024-04-05T03:09:01.997 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.error_if_exists: 0 2024-04-05T03:09:01.997 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.create_if_missing: 0 2024-04-05T03:09:01.997 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.paranoid_checks: 1 2024-04-05T03:09:01.997 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.env: 0x5599895bec60 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.fs: PosixFileSystem 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.info_log: 0x55998a397ae0 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.statistics: (nil) 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.use_fsync: 0 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_log_file_size: 0 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T03:09:02.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.allow_fallocate: 1 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.use_direct_reads: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.db_log_dir: 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.wal_dir: 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T03:09:02.280 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.write_buffer_manager: 0x55998a36a820 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T03:09:02.281 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.unordered_write: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.row_cache: None 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.wal_filter: None 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.two_write_queues: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.wal_compression: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.atomic_flush: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T03:09:02.282 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.log_readahead_size: 0 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_background_jobs: 2 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_background_compactions: -1 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_subcompactions: 1 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T03:09:02.283 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_open_files: -1 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_background_flushes: -1 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Compression algorithms supported: 2024-04-05T03:09:02.284 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kZSTD supported: 0 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kXpressCompression supported: 0 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kZlibCompression supported: 1 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kSnappyCompression supported: 1 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kLZ4Compression supported: 1 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: kBZip2Compression supported: 0 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.merge_operator: 2024-04-05T03:09:02.285 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_filter: None 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55998a397be0) 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks: 1 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout: pin_top_level_index_and_filter: 1 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_type: 0 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_index_type: 0 2024-04-05T03:09:02.286 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_shortening: 1 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: checksum: 4 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: no_block_cache: 0 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache: 0x55998a39b090 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_name: BinnedLRUCache 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_options: 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: capacity : 536870912 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: num_shard_bits : 4 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: strict_capacity_limit : 0 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: high_pri_pool_ratio: 0.000 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_cache_compressed: (nil) 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: persistent_cache: (nil) 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size: 4096 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_size_deviation: 10 2024-04-05T03:09:02.287 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_restart_interval: 16 2024-04-05T03:09:02.288 INFO:journalctl@ceph.mon.b.smithi151.stdout: index_block_restart_interval: 1 2024-04-05T03:09:02.288 INFO:journalctl@ceph.mon.b.smithi151.stdout: metadata_block_size: 4096 2024-04-05T03:09:02.288 INFO:journalctl@ceph.mon.b.smithi151.stdout: partition_filters: 0 2024-04-05T03:09:02.288 INFO:journalctl@ceph.mon.b.smithi151.stdout: use_delta_encoding: 1 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: filter_policy: bloomfilter 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: whole_key_filtering: 1 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: verify_compression: 0 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: read_amp_bytes_per_bit: 0 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: format_version: 5 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_index_compression: 1 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: block_align: 0 2024-04-05T03:09:02.289 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_auto_readahead_size: 262144 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout: prepopulate_block_cache: 0 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout: initial_auto_readahead_size: 8192 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout: num_file_reads_for_auto_readahead: 2 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression: NoCompression 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.num_levels: 7 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T03:09:02.290 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T03:09:02.291 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T03:09:02.292 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.arena_block_size: 1048576 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T03:09:02.293 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.table_properties_collectors: 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.inplace_update_support: 0 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T03:09:02.294 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.bloom_locality: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.max_successive_merges: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.ttl: 2592000 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.enable_blob_files: false 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.min_blob_size: 0 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T03:09:02.295 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.blob_file_starting_level: 0 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13080, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0f86cd9d-4d3d-4794-b4ab-6c440e3c63ec 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286541995628, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:01 smithi151 ceph-mon[88672]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286542001911, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1350576, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13085, "largest_seqno": 14351, "table_properties": {"data_size": 1344983, "index_size": 3362, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10389, "raw_average_key_size": 22, "raw_value_size": 1334690, "raw_average_value_size": 2851, "num_data_blocks": 155, "num_entries": 468, "num_filter_entries": 468, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712286541, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0f86cd9d-4d3d-4794-b4ab-6c440e3c63ec", "db_session_id": "JJTU19BWO8I9N4GACQGH", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-05T03:09:02.296 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712286542002044, "job": 1, "event": "recovery_finished"} 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55998a48c000 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: DB pointer 0x55998a476000 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** DB Stats ** 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T03:09:02.297 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: L0 1/0 1.29 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: L6 1/0 8.31 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: Sum 2/0 9.59 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 491.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** Compaction Stats [default] ** 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T03:09:02.298 INFO:journalctl@ceph.mon.b.smithi151.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 491.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: Cumulative compaction: 0.00 GB write, 149.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: Interval compaction: 0.00 GB write, 149.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T03:09:02.299 INFO:journalctl@ceph.mon.b.smithi151.stdout: Block cache BinnedLRUCache@0x55998a39b090#2 capacity: 512.00 MB usage: 5.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: starting mon.b rank 1 at public addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] at bind addrs [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???) e3 preinit fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???).mds e8 new map 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???).mds e8 print_map 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: e8 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: legacy client fscid: 1 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: Filesystem 'foo' (1) 2024-04-05T03:09:02.300 INFO:journalctl@ceph.mon.b.smithi151.stdout: fs_name foo 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: epoch 7 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: created 2024-04-05T03:02:48.450117+0000 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: modified 2024-04-05T03:03:33.860260+0000 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: tableserver 0 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: root 0 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: session_timeout 60 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: session_autoclose 300 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_file_size 1099511627776 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_xattr_size 65536 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: required_client_features {} 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: last_failure 0 2024-04-05T03:09:02.301 INFO:journalctl@ceph.mon.b.smithi151.stdout: last_failure_osd_epoch 0 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: max_mds 1 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: in 0 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: up {0=14718} 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: failed 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: damaged 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: stopped 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: data_pools [4] 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: metadata_pool 3 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: inline_data disabled 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: balancer 2024-04-05T03:09:02.302 INFO:journalctl@ceph.mon.b.smithi151.stdout: bal_rank_mask -1 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: standby_count_wanted 1 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: qdb_cluster leader: 0 members: 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: [mds.foo.smithi033.rmzwzs{0:14718} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.33:6834/3766639503,v1:172.21.15.33:6835/3766639503] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: Standby daemons: 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout: [mds.foo.smithi151.cxpowy{-1:24535} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.151:6832/1823067873,v1:172.21.15.151:6833/1823067873] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???).osd e88 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 ceph-mon[88672]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 bash[88653]: 634be946ebb9a25aefab251da1f74329c86b43354fed1500f82fe92f4ead4737 2024-04-05T03:09:02.303 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:02 smithi151 systemd[1]: Started Ceph mon.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:09:07.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:09:07 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:09:07.010+0000 7f5169f0e700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-05T03:09:08.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:09:08.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mgrmap e38: y(active, since 106s), standbys: x 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mon.b calling monitor election 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mon.a calling monitor election 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mon.c calling monitor election 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mon.b calling monitor election 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mon.a calling monitor election 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: monmap epoch 4 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: last_changed 2024-04-05T03:09:03.436247+0000 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: min_mon_release 19 (squid) 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: election_strategy: 1 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:09:08.780 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: mgrmap e38: y(active, since 111s), standbys: x 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.781 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:08 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mgrmap e38: y(active, since 106s), standbys: x 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mon.b calling monitor election 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mon.a calling monitor election 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mon.c calling monitor election 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mon.b calling monitor election 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mon.a calling monitor election 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: monmap epoch 4 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: last_changed 2024-04-05T03:09:03.436247+0000 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: min_mon_release 19 (squid) 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: election_strategy: 1 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: mgrmap e38: y(active, since 111s), standbys: x 2024-04-05T03:09:08.983 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.984 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mgrmap e38: y(active, since 106s), standbys: x 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mon.b calling monitor election 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mon.a calling monitor election 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mon.c calling monitor election 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.985 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mon.b calling monitor election 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mon.a calling monitor election 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: monmap epoch 4 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: last_changed 2024-04-05T03:09:03.436247+0000 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: created 2024-04-05T02:56:01.729844+0000 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: min_mon_release 19 (squid) 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: election_strategy: 1 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-04-05T03:09:08.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: 1: [v2:172.21.15.151:3300/0,v1:172.21.15.151:6789/0] mon.b 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: fsmap foo:1 {0=foo.smithi033.rmzwzs=up:active} 1 up:standby 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: mgrmap e38: y(active, since 111s), standbys: x 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: daemon grafana.a on smithi151 is in error state 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:08.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:08 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:09.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:09 smithi151 ceph-mon[88672]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:09.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:09 smithi033 ceph-mon[97738]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:09.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:09 smithi033 ceph-mon[101783]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:10.528 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:10 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:10.087Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", public_addr=\"172.21.15.33\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.33\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-04-05T03:09:11.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:09:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:09:10] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2024-04-05T03:09:11.855 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:11 smithi151 ceph-mon[88672]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:11.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:11 smithi033 ceph-mon[101783]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:11.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:11 smithi033 ceph-mon[97738]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:13.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:13 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:13.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:13 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:13.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:13 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:13.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:13 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:13 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:13.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:13 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:14.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:14 smithi151 ceph-mon[88672]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:14.363 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:14 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:14.364 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:14 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:14.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:14 smithi033 ceph-mon[97738]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:14.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:14 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:14.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:14 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:14.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:14 smithi033 ceph-mon[101783]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:14.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:14 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:14.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:14 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:16.390 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:16 smithi151 ceph-mon[88672]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:16.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:16 smithi033 ceph-mon[97738]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:16 smithi033 ceph-mon[101783]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:17.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:17.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:17.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[97738]: Reconfiguring mon.a (monmap changed)... 2024-04-05T03:09:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[97738]: Reconfiguring daemon mon.a on smithi033 2024-04-05T03:09:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[101783]: Reconfiguring mon.a (monmap changed)... 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[101783]: Reconfiguring daemon mon.a on smithi033 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:09:18.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:18.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:09:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-mon[88672]: Reconfiguring mon.a (monmap changed)... 2024-04-05T03:09:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-mon[88672]: Reconfiguring daemon mon.a on smithi033 2024-04-05T03:09:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:09:18.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:09:18.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:19.709 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:19 smithi033 ceph-mon[97738]: Reconfiguring mgr.y (monmap changed)... 2024-04-05T03:09:19.709 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:19 smithi033 ceph-mon[97738]: Reconfiguring daemon mgr.y on smithi033 2024-04-05T03:09:19.709 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:19 smithi033 ceph-mon[97738]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:19.709 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:19 smithi033 ceph-mon[101783]: Reconfiguring mgr.y (monmap changed)... 2024-04-05T03:09:19.709 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:19 smithi033 ceph-mon[101783]: Reconfiguring daemon mgr.y on smithi033 2024-04-05T03:09:19.709 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:19 smithi033 ceph-mon[101783]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:19 smithi151 ceph-mon[88672]: Reconfiguring mgr.y (monmap changed)... 2024-04-05T03:09:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:19 smithi151 ceph-mon[88672]: Reconfiguring daemon mgr.y on smithi033 2024-04-05T03:09:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:19 smithi151 ceph-mon[88672]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:20.962 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:09:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:09:20] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.43.0" 2024-04-05T03:09:21.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:21.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:21.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: Reconfiguring mon.c (monmap changed)... 2024-04-05T03:09:21.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: Reconfiguring daemon mon.c on smithi033 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[101783]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: Reconfiguring mon.c (monmap changed)... 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:21.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: Reconfiguring daemon mon.c on smithi033 2024-04-05T03:09:21.232 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:21 smithi033 ceph-mon[97738]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: Reconfiguring mon.c (monmap changed)... 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: Reconfiguring daemon mon.c on smithi033 2024-04-05T03:09:21.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:21 smithi151 ceph-mon[88672]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:22.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:22.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[101783]: Reconfiguring osd.0 (monmap changed)... 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.0 on smithi033 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[101783]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[97738]: Reconfiguring osd.0 (monmap changed)... 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.0 on smithi033 2024-04-05T03:09:22.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:22 smithi033 ceph-mon[97738]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:22 smithi151 ceph-mon[88672]: Reconfiguring osd.0 (monmap changed)... 2024-04-05T03:09:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:09:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:22 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.0 on smithi033 2024-04-05T03:09:23.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:22 smithi151 ceph-mon[88672]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[97738]: Reconfiguring osd.1 (monmap changed)... 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.1 on smithi033 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[97738]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[101783]: Reconfiguring osd.1 (monmap changed)... 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:09:24.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:24.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.1 on smithi033 2024-04-05T03:09:24.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:24 smithi033 ceph-mon[101783]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:24 smithi151 ceph-mon[88672]: Reconfiguring osd.1 (monmap changed)... 2024-04-05T03:09:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:09:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:24.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:24 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.1 on smithi033 2024-04-05T03:09:24.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:24 smithi151 ceph-mon[88672]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[101783]: Reconfiguring osd.2 (monmap changed)... 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.2 on smithi033 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[97738]: Reconfiguring osd.2 (monmap changed)... 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:26.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:26 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.2 on smithi033 2024-04-05T03:09:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:26 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:26 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:26 smithi151 ceph-mon[88672]: Reconfiguring osd.2 (monmap changed)... 2024-04-05T03:09:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:26 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:09:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:26 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:26.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:26 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.2 on smithi033 2024-04-05T03:09:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:27 smithi151 ceph-mon[88672]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:27 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:27 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:27 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:09:27.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:27 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[97738]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[101783]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:27.691 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:09:27.692 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:27 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:28.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:28 smithi151 ceph-mon[88672]: Reconfiguring osd.3 (monmap changed)... 2024-04-05T03:09:28.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:28 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.3 on smithi033 2024-04-05T03:09:28.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:28 smithi033 ceph-mon[97738]: Reconfiguring osd.3 (monmap changed)... 2024-04-05T03:09:28.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:28 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.3 on smithi033 2024-04-05T03:09:28.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:28 smithi033 ceph-mon[101783]: Reconfiguring osd.3 (monmap changed)... 2024-04-05T03:09:28.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:28 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.3 on smithi033 2024-04-05T03:09:29.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[101783]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[97738]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:29.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:29.527 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:09:29.527 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:29 smithi151 ceph-mon[88672]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi033.rmzwzs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:09:29.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:30.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[97738]: Reconfiguring mds.foo.smithi033.rmzwzs (monmap changed)... 2024-04-05T03:09:30.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[97738]: Reconfiguring daemon mds.foo.smithi033.rmzwzs on smithi033 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[101783]: Reconfiguring mds.foo.smithi033.rmzwzs (monmap changed)... 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[101783]: Reconfiguring daemon mds.foo.smithi033.rmzwzs on smithi033 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:30.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:30.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:30 smithi151 ceph-mon[88672]: Reconfiguring mds.foo.smithi033.rmzwzs (monmap changed)... 2024-04-05T03:09:30.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:30 smithi151 ceph-mon[88672]: Reconfiguring daemon mds.foo.smithi033.rmzwzs on smithi033 2024-04-05T03:09:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T03:09:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T03:09:30.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:30.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:09:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:09:30] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.43.0" 2024-04-05T03:09:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:31 smithi151 ceph-mon[88672]: Reconfiguring mon.b (monmap changed)... 2024-04-05T03:09:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:31 smithi151 ceph-mon[88672]: Reconfiguring daemon mon.b on smithi151 2024-04-05T03:09:31.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:31 smithi151 ceph-mon[88672]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:31 smithi033 ceph-mon[97738]: Reconfiguring mon.b (monmap changed)... 2024-04-05T03:09:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:31 smithi033 ceph-mon[97738]: Reconfiguring daemon mon.b on smithi151 2024-04-05T03:09:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:31 smithi033 ceph-mon[97738]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:31 smithi033 ceph-mon[101783]: Reconfiguring mon.b (monmap changed)... 2024-04-05T03:09:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:31 smithi033 ceph-mon[101783]: Reconfiguring daemon mon.b on smithi151 2024-04-05T03:09:31.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:31 smithi033 ceph-mon[101783]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:32.550 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:32.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:32.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:32.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:09:32.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:09:32.551 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:32.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T03:09:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:33.420 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:33 smithi151 ceph-mon[88672]: Reconfiguring mgr.x (monmap changed)... 2024-04-05T03:09:33.420 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:33 smithi151 ceph-mon[88672]: Reconfiguring daemon mgr.x on smithi151 2024-04-05T03:09:33.420 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:33 smithi151 ceph-mon[88672]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:33 smithi033 ceph-mon[97738]: Reconfiguring mgr.x (monmap changed)... 2024-04-05T03:09:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:33 smithi033 ceph-mon[97738]: Reconfiguring daemon mgr.x on smithi151 2024-04-05T03:09:33.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:33 smithi033 ceph-mon[97738]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:33 smithi033 ceph-mon[101783]: Reconfiguring mgr.x (monmap changed)... 2024-04-05T03:09:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:33 smithi033 ceph-mon[101783]: Reconfiguring daemon mgr.x on smithi151 2024-04-05T03:09:33.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:33 smithi033 ceph-mon[101783]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T03:09:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:34.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:34.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:34.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T03:09:34.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:34.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:34.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:34.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T03:09:34.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:35 smithi151 ceph-mon[88672]: Reconfiguring osd.4 (monmap changed)... 2024-04-05T03:09:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:35 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.4 on smithi151 2024-04-05T03:09:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:35 smithi151 ceph-mon[88672]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:35 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:35 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:35 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:09:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:35 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:35.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[97738]: Reconfiguring osd.4 (monmap changed)... 2024-04-05T03:09:35.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.4 on smithi151 2024-04-05T03:09:35.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[97738]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[101783]: Reconfiguring osd.4 (monmap changed)... 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.4 on smithi151 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[101783]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:09:35.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:35 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:36 smithi151 ceph-mon[88672]: Reconfiguring osd.5 (monmap changed)... 2024-04-05T03:09:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:36 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.5 on smithi151 2024-04-05T03:09:36.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:36 smithi033 ceph-mon[97738]: Reconfiguring osd.5 (monmap changed)... 2024-04-05T03:09:36.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:36 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.5 on smithi151 2024-04-05T03:09:36.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:36 smithi033 ceph-mon[101783]: Reconfiguring osd.5 (monmap changed)... 2024-04-05T03:09:36.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:36 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.5 on smithi151 2024-04-05T03:09:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:37 smithi151 ceph-mon[88672]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:37 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:37 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:37 smithi151 ceph-mon[88672]: Reconfiguring osd.6 (monmap changed)... 2024-04-05T03:09:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:37 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:09:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:37 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:37.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:37 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.6 on smithi151 2024-04-05T03:09:37.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[97738]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:37.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:37.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[97738]: Reconfiguring osd.6 (monmap changed)... 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.6 on smithi151 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[101783]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[101783]: Reconfiguring osd.6 (monmap changed)... 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:37.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:37 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.6 on smithi151 2024-04-05T03:09:39.003 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:09:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:39 smithi151 ceph-mon[88672]: Reconfiguring osd.7 (monmap changed)... 2024-04-05T03:09:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:09:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:39 smithi151 ceph-mon[88672]: Reconfiguring daemon osd.7 on smithi151 2024-04-05T03:09:39.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:39 smithi151 ceph-mon[88672]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:39.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:39.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[97738]: Reconfiguring osd.7 (monmap changed)... 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[97738]: Reconfiguring daemon osd.7 on smithi151 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[97738]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[101783]: Reconfiguring osd.7 (monmap changed)... 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[101783]: Reconfiguring daemon osd.7 on smithi151 2024-04-05T03:09:39.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:39 smithi033 ceph-mon[101783]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:41.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:40 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:41.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:40 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:41.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:40 smithi151 ceph-mon[88672]: Reconfiguring mds.foo.smithi151.cxpowy (monmap changed)... 2024-04-05T03:09:41.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:40 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:09:41.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:40 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:41.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:40 smithi151 ceph-mon[88672]: Reconfiguring daemon mds.foo.smithi151.cxpowy on smithi151 2024-04-05T03:09:41.030 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:40 smithi151 ceph-mon[88672]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:09:40] "GET /metrics HTTP/1.1" 200 34039 "" "Prometheus/2.43.0" 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[97738]: Reconfiguring mds.foo.smithi151.cxpowy (monmap changed)... 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[97738]: Reconfiguring daemon mds.foo.smithi151.cxpowy on smithi151 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[97738]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:41.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:41.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:41.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[101783]: Reconfiguring mds.foo.smithi151.cxpowy (monmap changed)... 2024-04-05T03:09:41.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi151.cxpowy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:09:41.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:41.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[101783]: Reconfiguring daemon mds.foo.smithi151.cxpowy on smithi151 2024-04-05T03:09:41.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:40 smithi033 ceph-mon[101783]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:42.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:42.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:09:44.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:44 smithi033 ceph-mon[97738]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:44.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:44 smithi033 ceph-mon[101783]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:44.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:44 smithi151 ceph-mon[88672]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:46.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:46 smithi033 ceph-mon[97738]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:46.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:46 smithi033 ceph-mon[101783]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:46.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:46 smithi151 ceph-mon[88672]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:48.694 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:48 smithi151 ceph-mon[88672]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:48.694 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:09:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:48 smithi033 ceph-mon[97738]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:48.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:48 smithi033 ceph-mon[101783]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:50.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:50 smithi033 ceph-mon[97738]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:50.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:50 smithi033 ceph-mon[101783]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:50.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:50 smithi151 ceph-mon[88672]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:51.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:09:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:09:50] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.43.0" 2024-04-05T03:09:52.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:52 smithi033 ceph-mon[97738]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:52.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:52 smithi033 ceph-mon[101783]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:52.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:52 smithi151 ceph-mon[88672]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:54.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:54 smithi033 ceph-mon[97738]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:54.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:54 smithi033 ceph-mon[101783]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:54.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:54 smithi151 ceph-mon[88672]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:56 smithi033 ceph-mon[101783]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:56 smithi033 ceph-mon[97738]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:56 smithi151 ceph-mon[88672]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:58.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:09:58 smithi033 ceph-mon[101783]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:58.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:09:58 smithi033 ceph-mon[97738]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:58.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:09:58 smithi151 ceph-mon[88672]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:58.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:09:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:09:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:10:00.343 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade resume' 2024-04-05T03:10:00.611 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:00.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:00 smithi033 ceph-mon[101783]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:00.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:00 smithi033 ceph-mon[101783]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:10:00.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:00 smithi033 ceph-mon[97738]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:00.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:00 smithi033 ceph-mon[97738]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:10:00.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:00 smithi151 ceph-mon[88672]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:00.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:00 smithi151 ceph-mon[88672]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:10:00.995 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:10:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:10:00] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.43.0" 2024-04-05T03:10:02.655 INFO:teuthology.orchestra.run.smithi033.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:02.656 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:02 smithi033 ceph-mon[101783]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:02.656 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:02.656 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:02 smithi033 ceph-mon[97738]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:02.656 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:02.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:02 smithi151 ceph-mon[88672]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:02.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:03.264 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for all mons to upgrade' 2024-04-05T03:10:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[97738]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[97738]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:03.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:03.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[97738]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[101783]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[101783]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:03 smithi033 ceph-mon[101783]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:04.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:03 smithi151 ceph-mon[88672]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:03 smithi151 ceph-mon[88672]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:03 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:03 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:03 smithi151 ceph-mon[88672]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:04.175 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:06.394 INFO:teuthology.orchestra.run.smithi033.stdout:wait for all mons to upgrade 2024-04-05T03:10:06.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:06 smithi033 ceph-mon[97738]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:06.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:06 smithi033 ceph-mon[101783]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:06.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:06 smithi151 ceph-mon[88672]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:07.119 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-05T03:10:07.801 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:08.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:08 smithi033 ceph-mon[101783]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:08.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:08 smithi033 ceph-mon[97738]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:08.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:08 smithi151 ceph-mon[88672]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:08.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:10:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:10:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:10:10.027 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 3 2024-04-05T03:10:10.627 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[97738]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:10.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1088881671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:10:10.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:10.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:10.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[101783]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:10.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1088881671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:10:10.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:10.628 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:10.652 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:10:10.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:10 smithi151 ceph-mon[88672]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:10.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:10 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1088881671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T03:10:10.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:10 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:10.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:10 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:10.920 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:10.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:10:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:10:10] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-04-05T03:10:11.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:10:11.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.732 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:10:11.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:11.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:11.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:11.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:11.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[97738]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[97738]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[97738]: Upgrade: Setting container_image for all mon 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[97738]: Upgrade: Setting container_image for all crash 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[97738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[97738]: Upgrade: osd.0 is safe to restart 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[101783]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[101783]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.634 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[101783]: Upgrade: Setting container_image for all mon 2024-04-05T03:10:12.634 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[101783]: Upgrade: Setting container_image for all crash 2024-04-05T03:10:12.634 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[101783]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:10:12.634 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:12 smithi033 ceph-mon[101783]: Upgrade: osd.0 is safe to restart 2024-04-05T03:10:12.760 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (4m) 2s ago 8m 26.9M - 0.25.0 c8568f914cd2 a9cf7c76c519 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi151 *:3000 error 60s ago 7m - - 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi033.rmzwzs smithi033 running (6m) 2s ago 6m 16.0M - 17.2.7-805-g6fda41ff a49113eacc58 c7d056f2104c 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi151.cxpowy smithi151 running (6m) 60s ago 6m 16.4M - 17.2.7-805-g6fda41ff a49113eacc58 0d112d294451 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi151 *:8443,9283,8765 running (3m) 60s ago 12m 450M - 19.0.0-2013-g0028525c d8c35eaf3c25 090d83bbd680 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (3m) 2s ago 14m 501M - 19.0.0-2013-g0028525c d8c35eaf3c25 a6e11bc7dc6c 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (3m) 2s ago 14m 45.4M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 7657e117c64f 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi151 running (71s) 60s ago 12m 21.3M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 634be946ebb9 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (92s) 2s ago 12m 34.0M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 fb77798ae736 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (4m) 2s ago 7m 15.5M - 1.5.0 0da6a335fe13 6da1bdc228ef 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi151 *:9100 running (3m) 60s ago 7m 16.6M - 1.5.0 0da6a335fe13 18b78d165e69 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (11m) 2s ago 11m 49.6M 2481M 17.2.7-805-g6fda41ff a49113eacc58 d0c1cb9b20a4 2024-04-05T03:10:12.761 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (11m) 2s ago 11m 51.2M 2481M 17.2.7-805-g6fda41ff a49113eacc58 279958cf423a 2024-04-05T03:10:12.762 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (11m) 2s ago 11m 46.9M 2481M 17.2.7-805-g6fda41ff a49113eacc58 309a3fe45561 2024-04-05T03:10:12.762 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (10m) 2s ago 10m 44.1M 2481M 17.2.7-805-g6fda41ff a49113eacc58 2b3b4d99eb65 2024-04-05T03:10:12.762 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi151 running (10m) 60s ago 10m 45.2M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b7f73016d7ed 2024-04-05T03:10:12.762 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi151 running (9m) 60s ago 9m 49.2M 2480M 17.2.7-805-g6fda41ff a49113eacc58 e2607db1cdd0 2024-04-05T03:10:12.762 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi151 running (9m) 60s ago 9m 48.6M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2063f47e19f2 2024-04-05T03:10:12.762 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi151 running (9m) 60s ago 9m 49.5M 2480M 17.2.7-805-g6fda41ff a49113eacc58 164624ba7a6d 2024-04-05T03:10:12.762 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi151 *:9095 running (3m) 60s ago 8m 61.2M - 2.43.0 a07b618ecd1d c6c02f0b66d8 2024-04-05T03:10:12.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:12 smithi151 ceph-mon[88672]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:12 smithi151 ceph-mon[88672]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:12 smithi151 ceph-mon[88672]: Upgrade: Setting container_image for all mon 2024-04-05T03:10:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:12 smithi151 ceph-mon[88672]: Upgrade: Setting container_image for all crash 2024-04-05T03:10:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:12 smithi151 ceph-mon[88672]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:10:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:12 smithi151 ceph-mon[88672]: Upgrade: osd.0 is safe to restart 2024-04-05T03:10:14.343 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T03:10:14.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[97738]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:14.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[97738]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[101783]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[101783]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:10:14.603 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:14 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:14 smithi151 ceph-mon[88672]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:14 smithi151 ceph-mon[88672]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:14 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:14 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:10:14.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:14 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:15.660 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:15.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:15 smithi033 ceph-mon[97738]: Upgrade: Updating osd.0 2024-04-05T03:10:15.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:15 smithi033 ceph-mon[97738]: Deploying daemon osd.0 on smithi033 2024-04-05T03:10:15.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:15 smithi033 ceph-mon[101783]: Upgrade: Updating osd.0 2024-04-05T03:10:15.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:15 smithi033 ceph-mon[101783]: Deploying daemon osd.0 on smithi033 2024-04-05T03:10:15.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:15 smithi151 ceph-mon[88672]: Upgrade: Updating osd.0 2024-04-05T03:10:15.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:15 smithi151 ceph-mon[88672]: Deploying daemon osd.0 on smithi033 2024-04-05T03:10:16.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:16 smithi033 ceph-mon[97738]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:16 smithi033 ceph-mon[101783]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:16.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:16 smithi151 ceph-mon[88672]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:17.559 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:17.559 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:17.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:18.185 INFO:teuthology.orchestra.run.smithi033.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:18.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:18 smithi033 ceph-mon[101783]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:18.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:18 smithi033 ceph-mon[97738]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:18.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:18 smithi151 ceph-mon[88672]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:18.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:18.778 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:10:18 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:10:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:10:19.230 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:18 smithi033 systemd[1]: Stopping Ceph osd.0 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:10:19.269 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'sleep 60' 2024-04-05T03:10:19.555 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:19 smithi033 ceph-mon[101783]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:19.555 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:19 smithi033 ceph-mon[101783]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:19.556 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:19 smithi033 ceph-mon[97738]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:19.556 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:19 smithi033 ceph-mon[97738]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:19.622 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:19 smithi151 ceph-mon[88672]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:10:19.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:19 smithi151 ceph-mon[88672]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:19.867 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:19 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[43943]: 2024-04-05T03:10:19.554+0000 7f18e2b3e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:10:19.867 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:19 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[43943]: 2024-04-05T03:10:19.554+0000 7f18e2b3e700 -1 osd.0 88 *** Got signal Terminated *** 2024-04-05T03:10:19.867 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:19 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[43943]: 2024-04-05T03:10:19.554+0000 7f18e2b3e700 -1 osd.0 88 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:10:20.717 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:20 smithi033 ceph-mon[97738]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:20.717 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:20 smithi033 ceph-mon[97738]: osd.0 marked itself down and dead 2024-04-05T03:10:20.717 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:20 smithi033 ceph-mon[101783]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:20.718 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:20 smithi033 ceph-mon[101783]: osd.0 marked itself down and dead 2024-04-05T03:10:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:20 smithi151 ceph-mon[88672]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:20.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:20 smithi151 ceph-mon[88672]: osd.0 marked itself down and dead 2024-04-05T03:10:20.972 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:10:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:10:20] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.43.0" 2024-04-05T03:10:21.230 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:20 smithi033 podman[108678]: 2024-04-05 03:10:20.972591433 +0000 UTC m=+1.538106026 container died d0c1cb9b20a490f4824e400ed77c91cb9c269d29770c63f1fe99d5e63bb4bde4 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-05T03:10:21.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[97738]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:10:21.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[97738]: failed to encode map e89 with expected crc 2024-04-05T03:10:21.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[97738]: failed to encode map e89 with expected crc 2024-04-05T03:10:21.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[97738]: osdmap e89: 8 total, 7 up, 8 in 2024-04-05T03:10:21.625 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[101783]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:10:21.625 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[101783]: failed to encode map e89 with expected crc 2024-04-05T03:10:21.625 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[101783]: failed to encode map e89 with expected crc 2024-04-05T03:10:21.625 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:21 smithi033 ceph-mon[101783]: osdmap e89: 8 total, 7 up, 8 in 2024-04-05T03:10:21.625 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:21 smithi033 podman[108678]: 2024-04-05 03:10:21.35697181 +0000 UTC m=+1.922486395 container cleanup d0c1cb9b20a490f4824e400ed77c91cb9c269d29770c63f1fe99d5e63bb4bde4 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-05T03:10:21.625 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:21 smithi033 bash[108678]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0 2024-04-05T03:10:21.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:21 smithi151 ceph-mon[88672]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:10:21.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:21 smithi151 ceph-mon[88672]: failed to encode map e89 with expected crc 2024-04-05T03:10:21.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:21 smithi151 ceph-mon[88672]: failed to encode map e89 with expected crc 2024-04-05T03:10:21.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:21 smithi151 ceph-mon[88672]: osdmap e89: 8 total, 7 up, 8 in 2024-04-05T03:10:21.980 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:21 smithi033 podman[108792]: 2024-04-05 03:10:21.62528733 +0000 UTC m=+0.653639286 container remove d0c1cb9b20a490f4824e400ed77c91cb9c269d29770c63f1fe99d5e63bb4bde4 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-05T03:10:22.312 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:22 smithi033 podman[108887]: 2024-04-05 03:10:21.920667074 +0000 UTC m=+0.017990456 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:22.312 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:22 smithi033 podman[108887]: 2024-04-05 03:10:22.030292987 +0000 UTC m=+0.127616391 container create d30a28621cfab91103b6211676a148c7126621b9e5d7b365b7f6421b33a8e002 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:10:22.312 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:22 smithi033 podman[108887]: 2024-04-05 03:10:22.238624614 +0000 UTC m=+0.335947986 container init d30a28621cfab91103b6211676a148c7126621b9e5d7b365b7f6421b33a8e002 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, RELEASE=squid-0028525, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS) 2024-04-05T03:10:22.312 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:22 smithi033 podman[108887]: 2024-04-05 03:10:22.241984421 +0000 UTC m=+0.339307786 container start d30a28621cfab91103b6211676a148c7126621b9e5d7b365b7f6421b33a8e002 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-0028525, io.buildah.version=1.33.5) 2024-04-05T03:10:22.312 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:22 smithi033 podman[108887]: 2024-04-05 03:10:22.313052355 +0000 UTC m=+0.410375725 container attach d30a28621cfab91103b6211676a148c7126621b9e5d7b365b7f6421b33a8e002 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-05T03:10:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: pgmap v104: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: osdmap e90: 8 total, 7 up, 8 in 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[101783]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.731 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:22 smithi033 podman[108887]: 2024-04-05 03:10:22.537423978 +0000 UTC m=+0.634747349 container died d30a28621cfab91103b6211676a148c7126621b9e5d7b365b7f6421b33a8e002 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, org.label-schema.build-date=20240403, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-05T03:10:22.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: pgmap v104: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: osdmap e90: 8 total, 7 up, 8 in 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:22 smithi033 ceph-mon[97738]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: pgmap v104: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: osdmap e90: 8 total, 7 up, 8 in 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:22.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:22 smithi151 ceph-mon[88672]: failed to encode map e90 with expected crc 2024-04-05T03:10:23.480 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 podman[108913]: 2024-04-05 03:10:23.109102604 +0000 UTC m=+0.566532362 container remove d30a28621cfab91103b6211676a148c7126621b9e5d7b365b7f6421b33a8e002 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:10:23.480 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.0.service: Deactivated successfully. 2024-04-05T03:10:23.480 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 systemd[1]: Stopped Ceph osd.0 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:10:23.480 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.0.service: Consumed 5.426s CPU time. 2024-04-05T03:10:23.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 systemd[1]: Starting Ceph osd.0 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:10:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:23 smithi151 ceph-mon[88672]: pgmap v106: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:23.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:23 smithi151 ceph-mon[88672]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-04-05T03:10:23.778 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:23 smithi033 ceph-mon[101783]: pgmap v106: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:23.778 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:23 smithi033 ceph-mon[101783]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-04-05T03:10:23.778 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:23 smithi033 ceph-mon[97738]: pgmap v106: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:23.779 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:23 smithi033 ceph-mon[97738]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-04-05T03:10:23.779 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 podman[109018]: 2024-04-05 03:10:23.466641736 +0000 UTC m=+0.018645177 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:23.779 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 podman[109018]: 2024-04-05 03:10:23.585273685 +0000 UTC m=+0.137277168 container create cc40e47eaf07247f3d01d2bb692b1dbc024d6a671e905d8b950e440ad4b59068 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-0028525, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:10:24.228 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 podman[109018]: 2024-04-05 03:10:23.779242917 +0000 UTC m=+0.331246357 container init cc40e47eaf07247f3d01d2bb692b1dbc024d6a671e905d8b950e440ad4b59068 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-0028525, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-05T03:10:24.228 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 podman[109018]: 2024-04-05 03:10:23.782558432 +0000 UTC m=+0.334561873 container start cc40e47eaf07247f3d01d2bb692b1dbc024d6a671e905d8b950e440ad4b59068 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-05T03:10:24.228 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:23 smithi033 podman[109018]: 2024-04-05 03:10:23.854217501 +0000 UTC m=+0.406220952 container attach cc40e47eaf07247f3d01d2bb692b1dbc024d6a671e905d8b950e440ad4b59068 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403) 2024-04-05T03:10:24.480 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate[109032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate[109032]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 bash[109018]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 bash[109018]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 bash[109018]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 bash[109018]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 bash[109018]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 bash[109018]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 bash[109018]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate[109032]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate[109032]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate[109032]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate[109032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate[109032]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-05T03:10:24.481 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 podman[109018]: 2024-04-05 03:10:24.252470502 +0000 UTC m=+0.804473947 container died cc40e47eaf07247f3d01d2bb692b1dbc024d6a671e905d8b950e440ad4b59068 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:10:24.980 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:24 smithi033 podman[109156]: 2024-04-05 03:10:24.799971033 +0000 UTC m=+0.542275102 container remove cc40e47eaf07247f3d01d2bb692b1dbc024d6a671e905d8b950e440ad4b59068 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-05T03:10:25.319 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 podman[109220]: 2024-04-05 03:10:24.920044045 +0000 UTC m=+0.018236626 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:10:25.319 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 podman[109220]: 2024-04-05 03:10:25.034504757 +0000 UTC m=+0.132697313 container create 6a19fd0140d9f1f4cc63d4c79cf41b2f07a76c9038dcb9a6f48c4d2db2f6436c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, GIT_CLEAN=True, RELEASE=squid-0028525, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2) 2024-04-05T03:10:25.319 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 podman[109220]: 2024-04-05 03:10:25.228600524 +0000 UTC m=+0.326793085 container init 6a19fd0140d9f1f4cc63d4c79cf41b2f07a76c9038dcb9a6f48c4d2db2f6436c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, RELEASE=squid-0028525, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:10:25.319 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 podman[109220]: 2024-04-05 03:10:25.231703503 +0000 UTC m=+0.329896060 container start 6a19fd0140d9f1f4cc63d4c79cf41b2f07a76c9038dcb9a6f48c4d2db2f6436c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2) 2024-04-05T03:10:25.319 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 ceph-osd[109240]: -- 172.21.15.33:0/590185956 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a86d59b680 con 0x55a86d565000 2024-04-05T03:10:25.730 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 bash[109220]: 6a19fd0140d9f1f4cc63d4c79cf41b2f07a76c9038dcb9a6f48c4d2db2f6436c 2024-04-05T03:10:25.730 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 systemd[1]: Started Ceph osd.0 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:10:26.230 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:25 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[109236]: 2024-04-05T03:10:25.770+0000 7fb3fad027c0 -1 Falling back to public interface 2024-04-05T03:10:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[97738]: pgmap v107: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[101783]: pgmap v107: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:26.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:26 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:26 smithi151 ceph-mon[88672]: pgmap v107: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:26 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:26 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:26.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:26 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:28.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:28 smithi151 ceph-mon[88672]: pgmap v108: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:28.679 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:28 smithi033 ceph-mon[97738]: pgmap v108: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:28.680 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:28 smithi033 ceph-mon[101783]: pgmap v108: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:28.680 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:28 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[109236]: 2024-04-05T03:10:28.507+0000 7fb3fad027c0 -1 osd.0 88 log_to_monitors true 2024-04-05T03:10:28.954 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:10:28 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:10:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:10:29.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:29 smithi033 ceph-mon[97738]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:10:29.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:29.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:29.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:29 smithi033 ceph-mon[101783]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:10:29.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:29.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:29.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:29 smithi151 ceph-mon[88672]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:10:29.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:29.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.230 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:10:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[109236]: 2024-04-05T03:10:29.852+0000 7fb3e8152700 -1 osd.0 88 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:10:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: pgmap v109: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T03:10:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: osdmap e91: 8 total, 7 up, 8 in 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: pgmap v109: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: osdmap e91: 8 total, 7 up, 8 in 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: pgmap v109: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: osdmap e91: 8 total, 7 up, 8 in 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: from='osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266]' entity='osd.0' 2024-04-05T03:10:30.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: failed to encode map e91 with expected crc 2024-04-05T03:10:30.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:30.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:31.080 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:10:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:10:30] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.43.0" 2024-04-05T03:10:31.700 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[101783]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:10:31.700 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[101783]: failed to encode map e92 with expected crc 2024-04-05T03:10:31.700 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[101783]: failed to encode map e92 with expected crc 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[101783]: osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266] boot 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[101783]: osdmap e92: 8 total, 8 up, 8 in 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[97738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[97738]: failed to encode map e92 with expected crc 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[97738]: failed to encode map e92 with expected crc 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[97738]: osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266] boot 2024-04-05T03:10:31.701 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[97738]: osdmap e92: 8 total, 8 up, 8 in 2024-04-05T03:10:31.702 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:31 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:10:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:31 smithi151 ceph-mon[88672]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:10:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:31 smithi151 ceph-mon[88672]: failed to encode map e92 with expected crc 2024-04-05T03:10:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:31 smithi151 ceph-mon[88672]: failed to encode map e92 with expected crc 2024-04-05T03:10:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:31 smithi151 ceph-mon[88672]: osd.0 [v2:172.21.15.33:6802/2467730266,v1:172.21.15.33:6803/2467730266] boot 2024-04-05T03:10:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:31 smithi151 ceph-mon[88672]: osdmap e92: 8 total, 8 up, 8 in 2024-04-05T03:10:31.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:31 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:10:32.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: pgmap v112: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: osdmap e93: 8 total, 8 up, 8 in 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[101783]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: pgmap v112: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: osdmap e93: 8 total, 8 up, 8 in 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:32 smithi033 ceph-mon[97738]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: pgmap v112: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: osdmap e93: 8 total, 8 up, 8 in 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: failed to encode map e93 with expected crc 2024-04-05T03:10:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:32 smithi151 ceph-mon[88672]: failed to encode map e93 with expected crc 2024-04-05T03:10:33.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:33 smithi033 ceph-mon[97738]: failed to encode map e93 with expected crc 2024-04-05T03:10:33.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:33 smithi033 ceph-mon[101783]: failed to encode map e93 with expected crc 2024-04-05T03:10:33.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:33 smithi151 ceph-mon[88672]: failed to encode map e93 with expected crc 2024-04-05T03:10:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: pgmap v114: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: pgmap v114: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:34.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: pgmap v114: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:10:34.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:10:34.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[97738]: pgmap v115: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[97738]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[97738]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[97738]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[101783]: pgmap v115: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[101783]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[101783]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:35 smithi033 ceph-mon[101783]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:35 smithi151 ceph-mon[88672]: pgmap v115: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:35 smithi151 ceph-mon[88672]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:35 smithi151 ceph-mon[88672]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:35 smithi151 ceph-mon[88672]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:10:36.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:36 smithi033 ceph-mon[97738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-05T03:10:36.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:36 smithi033 ceph-mon[101783]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-05T03:10:36.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:36 smithi151 ceph-mon[88672]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-05T03:10:37.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:37 smithi033 ceph-mon[97738]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:37 smithi033 ceph-mon[101783]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:37.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:37 smithi151 ceph-mon[88672]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:39.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:10:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:10:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:10:39.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:39 smithi033 ceph-mon[97738]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:39.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:39 smithi033 ceph-mon[101783]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:39.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:39 smithi151 ceph-mon[88672]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:10:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:10:40] "GET /metrics HTTP/1.1" 200 33956 "" "Prometheus/2.43.0" 2024-04-05T03:10:41.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:41 smithi033 ceph-mon[97738]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:41 smithi033 ceph-mon[101783]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:41.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:41 smithi151 ceph-mon[88672]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:42.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:42.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:42 smithi033 ceph-mon[97738]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:42.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:42.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:42 smithi033 ceph-mon[101783]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:43.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:43.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:42 smithi151 ceph-mon[88672]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:45.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:44 smithi151 ceph-mon[88672]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:45.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:44 smithi033 ceph-mon[97738]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:45.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:44 smithi033 ceph-mon[101783]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:47.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:46 smithi151 ceph-mon[88672]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:47.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:46 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:47.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:46 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:47.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:46 smithi033 ceph-mon[97738]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:47.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:46 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:47.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:46 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:47.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:46 smithi033 ceph-mon[101783]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:47.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:46 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:10:47.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:46 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:49.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:48 smithi151 ceph-mon[88672]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:49.028 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:10:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:10:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi033:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"smithi151.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T03:10:49.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:48 smithi033 ceph-mon[97738]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:49.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:48 smithi033 ceph-mon[101783]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:51.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:50 smithi151 ceph-mon[88672]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:51.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:50 smithi033 ceph-mon[97738]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:51.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:50 smithi033 ceph-mon[101783]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:51.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:10:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:10:50] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.43.0" 2024-04-05T03:10:53.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:52 smithi151 ceph-mon[88672]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:53.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:52 smithi033 ceph-mon[97738]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:53.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:52 smithi033 ceph-mon[101783]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:55.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:54 smithi151 ceph-mon[88672]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:55.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:54 smithi033 ceph-mon[97738]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:55.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:54 smithi033 ceph-mon[101783]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:57.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:56 smithi151 ceph-mon[88672]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:57.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:56 smithi033 ceph-mon[97738]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:57.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:56 smithi033 ceph-mon[101783]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:10:58 smithi151 ceph-mon[88672]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:59.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:10:58 smithi033 ceph-mon[97738]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:59.032 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:10:58 smithi033 ceph-mon[101783]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:00 smithi151 ceph-mon[88672]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:00 smithi033 ceph-mon[97738]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:00 smithi033 ceph-mon[101783]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:11:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:11:00] "GET /metrics HTTP/1.1" 200 34101 "" "Prometheus/2.43.0" 2024-04-05T03:11:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:01 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:02.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:01 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:02.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:01 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:03.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:02 smithi033 ceph-mon[97738]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:03.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:02 smithi033 ceph-mon[101783]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:03.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:02 smithi151 ceph-mon[88672]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:05.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:04 smithi033 ceph-mon[97738]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:05.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:04 smithi033 ceph-mon[101783]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:05.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:04 smithi151 ceph-mon[88672]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:07.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:06 smithi033 ceph-mon[97738]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:07.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:06 smithi033 ceph-mon[101783]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:07.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:06 smithi151 ceph-mon[88672]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:09.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:08 smithi033 ceph-mon[97738]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:09.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:08 smithi033 ceph-mon[101783]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:09.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:08 smithi151 ceph-mon[88672]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:11.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:10 smithi033 ceph-mon[97738]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:11.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:10 smithi033 ceph-mon[101783]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:11.231 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:11:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:11:10] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.43.0" 2024-04-05T03:11:11.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:10 smithi151 ceph-mon[88672]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:13.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:12 smithi033 ceph-mon[101783]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:13.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:12 smithi033 ceph-mon[97738]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:13.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:12 smithi151 ceph-mon[88672]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:15.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:14 smithi033 ceph-mon[97738]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:15.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:14 smithi033 ceph-mon[101783]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:15.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:14 smithi151 ceph-mon[88672]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:17.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:16 smithi033 ceph-mon[97738]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:17.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:16 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:17.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:16 smithi033 ceph-mon[101783]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:17.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:16 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:17.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:16 smithi151 ceph-mon[88672]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:17.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:16 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:19.080 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:18 smithi151 ceph-mon[88672]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:18 smithi033 ceph-mon[97738]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:19.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:18 smithi033 ceph-mon[101783]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:21.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:20 smithi033 ceph-mon[97738]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:21.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:11:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:11:20] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.43.0" 2024-04-05T03:11:21.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:20 smithi033 ceph-mon[101783]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:21.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:20 smithi151 ceph-mon[88672]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:22.424 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade resume' 2024-04-05T03:11:22.693 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:23.061 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:22 smithi033 ceph-mon[101783]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:23.062 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:22 smithi033 ceph-mon[97738]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:23.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:22 smithi151 ceph-mon[88672]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:24.552 INFO:teuthology.orchestra.run.smithi033.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.122 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:11:25.155 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for half of osds to upgrade' 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:25.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:25.279 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:11:25.861 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:26.220 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[97738]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[97738]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[97738]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[97738]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[97738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[97738]: Upgrade: osd.1 is safe to restart 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[101783]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[101783]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[101783]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[101783]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[101783]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:11:26.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:25 smithi033 ceph-mon[101783]: Upgrade: osd.1 is safe to restart 2024-04-05T03:11:26.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:25 smithi151 ceph-mon[88672]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:11:26.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:25 smithi151 ceph-mon[88672]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:11:26.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:25 smithi151 ceph-mon[88672]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:26.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:25 smithi151 ceph-mon[88672]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:26.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:25 smithi151 ceph-mon[88672]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:11:26.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:25 smithi151 ceph-mon[88672]: Upgrade: osd.1 is safe to restart 2024-04-05T03:11:27.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[97738]: Upgrade: Updating osd.1 2024-04-05T03:11:27.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:27.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:11:27.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:27.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[97738]: Deploying daemon osd.1 on smithi033 2024-04-05T03:11:27.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[101783]: Upgrade: Updating osd.1 2024-04-05T03:11:27.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:27.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:11:27.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:27.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:27 smithi033 ceph-mon[101783]: Deploying daemon osd.1 on smithi033 2024-04-05T03:11:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:27 smithi151 ceph-mon[88672]: Upgrade: Updating osd.1 2024-04-05T03:11:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:27 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:27 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:11:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:27 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:27.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:27 smithi151 ceph-mon[88672]: Deploying daemon osd.1 on smithi033 2024-04-05T03:11:28.252 INFO:teuthology.orchestra.run.smithi033.stdout:wait for half of osds to upgrade 2024-04-05T03:11:28.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:28 smithi033 ceph-mon[97738]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:28.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:28 smithi033 ceph-mon[101783]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:28.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:28 smithi151 ceph-mon[88672]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:29.690 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-05T03:11:30.220 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:30 smithi033 ceph-mon[97738]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:30 smithi033 ceph-mon[101783]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:30.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:30 smithi151 ceph-mon[88672]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:31.116 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:11:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:11:30] "GET /metrics HTTP/1.1" 200 34100 "" "Prometheus/2.43.0" 2024-04-05T03:11:31.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:31 smithi033 systemd[1]: Stopping Ceph osd.1 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:11:32.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:32 smithi033 ceph-mon[101783]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:32.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:32.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:32 smithi033 ceph-mon[97738]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:32.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:32 smithi151 ceph-mon[88672]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:32.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:32.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[49782]: 2024-04-05T03:11:32.764+0000 7f2740b08700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:11:32.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[49782]: 2024-04-05T03:11:32.764+0000 7f2740b08700 -1 osd.1 93 *** Got signal Terminated *** 2024-04-05T03:11:32.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:32 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[49782]: 2024-04-05T03:11:32.764+0000 7f2740b08700 -1 osd.1 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:11:33.692 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:33 smithi033 ceph-mon[97738]: osd.1 marked itself down and dead 2024-04-05T03:11:33.693 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:33 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3812417225' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:33.693 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:33 smithi033 ceph-mon[101783]: osd.1 marked itself down and dead 2024-04-05T03:11:33.693 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:33 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3812417225' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:33.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:33 smithi151 ceph-mon[88672]: osd.1 marked itself down and dead 2024-04-05T03:11:33.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:33 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3812417225' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:33.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:33 smithi033 podman[113922]: 2024-04-05 03:11:33.693738968 +0000 UTC m=+1.566532963 container died 279958cf423a0c1d252ee50a7962097d9f7c68845cc9da9d015425c8957f4dae (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-05T03:11:34.300 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:34 smithi033 podman[113922]: 2024-04-05 03:11:34.056297941 +0000 UTC m=+1.929091932 container cleanup 279958cf423a0c1d252ee50a7962097d9f7c68845cc9da9d015425c8957f4dae (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-04-05T03:11:34.300 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:34 smithi033 bash[113922]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[97738]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[97738]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[97738]: failed to encode map e94 with expected crc 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[97738]: failed to encode map e94 with expected crc 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[97738]: osdmap e94: 8 total, 7 up, 8 in 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[101783]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[101783]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:11:34.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[101783]: failed to encode map e94 with expected crc 2024-04-05T03:11:34.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[101783]: failed to encode map e94 with expected crc 2024-04-05T03:11:34.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:34 smithi033 ceph-mon[101783]: osdmap e94: 8 total, 7 up, 8 in 2024-04-05T03:11:34.678 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:34 smithi033 podman[113975]: 2024-04-05 03:11:34.324553982 +0000 UTC m=+0.630887988 container remove 279958cf423a0c1d252ee50a7962097d9f7c68845cc9da9d015425c8957f4dae (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:11:34.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:34 smithi151 ceph-mon[88672]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:34 smithi151 ceph-mon[88672]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:11:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:34 smithi151 ceph-mon[88672]: failed to encode map e94 with expected crc 2024-04-05T03:11:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:34 smithi151 ceph-mon[88672]: failed to encode map e94 with expected crc 2024-04-05T03:11:34.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:34 smithi151 ceph-mon[88672]: osdmap e94: 8 total, 7 up, 8 in 2024-04-05T03:11:34.969 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:34 smithi033 podman[114068]: 2024-04-05 03:11:34.581557036 +0000 UTC m=+0.018412803 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:11:34.969 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:34 smithi033 podman[114068]: 2024-04-05 03:11:34.709194439 +0000 UTC m=+0.146050220 container create 02fe5a03091a7b10674cc445473e430faa17c4e583bbef755caa932a5a785fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-05T03:11:34.969 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:34 smithi033 podman[114068]: 2024-04-05 03:11:34.970485252 +0000 UTC m=+0.407341021 container init 02fe5a03091a7b10674cc445473e430faa17c4e583bbef755caa932a5a785fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403) 2024-04-05T03:11:35.230 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:34 smithi033 podman[114068]: 2024-04-05 03:11:34.973854835 +0000 UTC m=+0.410710584 container start 02fe5a03091a7b10674cc445473e430faa17c4e583bbef755caa932a5a785fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , RELEASE=squid-0028525) 2024-04-05T03:11:35.230 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:35 smithi033 podman[114068]: 2024-04-05 03:11:35.070476063 +0000 UTC m=+0.507331816 container attach 02fe5a03091a7b10674cc445473e430faa17c4e583bbef755caa932a5a785fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD) 2024-04-05T03:11:35.711 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:35 smithi033 podman[114068]: 2024-04-05 03:11:35.272734147 +0000 UTC m=+0.709589881 container died 02fe5a03091a7b10674cc445473e430faa17c4e583bbef755caa932a5a785fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-0028525) 2024-04-05T03:11:35.711 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.711 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: osdmap e95: 8 total, 7 up, 8 in 2024-04-05T03:11:35.711 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.711 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.711 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.711 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.711 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/555884796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: osdmap e95: 8 total, 7 up, 8 in 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:35 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/555884796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:35.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: osdmap e95: 8 total, 7 up, 8 in 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:35.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:35 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/555884796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:35.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:35 smithi033 podman[114120]: 2024-04-05 03:11:35.883485388 +0000 UTC m=+0.605726850 container remove 02fe5a03091a7b10674cc445473e430faa17c4e583bbef755caa932a5a785fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, GIT_CLEAN=True, ceph=True, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:11:35.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:35 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.1.service: Deactivated successfully. 2024-04-05T03:11:35.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:35 smithi033 systemd[1]: Stopped Ceph osd.1 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:11:35.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:35 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.1.service: Consumed 6.716s CPU time. 2024-04-05T03:11:36.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:36.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[97738]: failed to encode map e95 with expected crc 2024-04-05T03:11:36.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[97738]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 2 active+undersized, 60 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:11:36.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[97738]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:36.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:36.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[101783]: failed to encode map e95 with expected crc 2024-04-05T03:11:36.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[101783]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 2 active+undersized, 60 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:11:36.449 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:36 smithi033 ceph-mon[101783]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:36.449 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:36 smithi033 systemd[1]: Starting Ceph osd.1 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:11:36.449 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:36 smithi033 podman[114226]: 2024-04-05 03:11:36.326676293 +0000 UTC m=+0.112537523 container create a1d8b120e093e5703b336d1e7ba3356ea6d3901d90dc37538bfec7df7e7fe612 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:11:36.449 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:36 smithi033 podman[114226]: 2024-04-05 03:11:36.232603414 +0000 UTC m=+0.018464654 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:11:36.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:36 smithi033 podman[114226]: 2024-04-05 03:11:36.562892706 +0000 UTC m=+0.348753937 container init a1d8b120e093e5703b336d1e7ba3356ea6d3901d90dc37538bfec7df7e7fe612 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-05T03:11:36.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:36 smithi033 podman[114226]: 2024-04-05 03:11:36.566094978 +0000 UTC m=+0.351956215 container start a1d8b120e093e5703b336d1e7ba3356ea6d3901d90dc37538bfec7df7e7fe612 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD) 2024-04-05T03:11:36.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:36 smithi033 podman[114226]: 2024-04-05 03:11:36.63788113 +0000 UTC m=+0.423742379 container attach a1d8b120e093e5703b336d1e7ba3356ea6d3901d90dc37538bfec7df7e7fe612 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-0028525, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:11:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:36 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:36 smithi151 ceph-mon[88672]: failed to encode map e95 with expected crc 2024-04-05T03:11:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:36 smithi151 ceph-mon[88672]: pgmap v152: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 2 active+undersized, 60 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:11:36.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:36 smithi151 ceph-mon[88672]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate[114241]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate[114241]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 bash[114226]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 bash[114226]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 bash[114226]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 bash[114226]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 bash[114226]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-05T03:11:37.407 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 bash[114226]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:11:37.408 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 bash[114226]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-05T03:11:37.408 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate[114241]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-05T03:11:37.408 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate[114241]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-05T03:11:37.408 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate[114241]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-05T03:11:37.408 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate[114241]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:11:37.408 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate[114241]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-05T03:11:37.408 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 podman[114226]: 2024-04-05 03:11:37.038905763 +0000 UTC m=+0.824766984 container died a1d8b120e093e5703b336d1e7ba3356ea6d3901d90dc37538bfec7df7e7fe612 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403) 2024-04-05T03:11:38.068 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 podman[114369]: 2024-04-05 03:11:37.767915315 +0000 UTC m=+0.723808942 container remove a1d8b120e093e5703b336d1e7ba3356ea6d3901d90dc37538bfec7df7e7fe612 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-05T03:11:38.069 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 podman[114459]: 2024-04-05 03:11:37.96897054 +0000 UTC m=+0.103498082 container create 22dcd72df86aae5032f9c56a2b1ebdde14628453523601ac09eaa722ff0a7460 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-0028525, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:11:38.069 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:37 smithi033 podman[114459]: 2024-04-05 03:11:37.883566685 +0000 UTC m=+0.018094225 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:11:38.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:38 smithi033 ceph-mon[97738]: pgmap v153: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 2 active+undersized, 60 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:11:38.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:38 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2814731685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:38.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:38 smithi033 ceph-mon[101783]: pgmap v153: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 2 active+undersized, 60 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:11:38.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:38 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2814731685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:38.480 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:38 smithi033 podman[114459]: 2024-04-05 03:11:38.171475967 +0000 UTC m=+0.306003505 container init 22dcd72df86aae5032f9c56a2b1ebdde14628453523601ac09eaa722ff0a7460 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403) 2024-04-05T03:11:38.481 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:38 smithi033 podman[114459]: 2024-04-05 03:11:38.174865077 +0000 UTC m=+0.309392615 container start 22dcd72df86aae5032f9c56a2b1ebdde14628453523601ac09eaa722ff0a7460 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-05T03:11:38.481 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:38 smithi033 ceph-osd[114478]: -- 172.21.15.33:0/366156563 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d47af43680 con 0x55d47af0d000 2024-04-05T03:11:38.481 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:38 smithi033 bash[114459]: 22dcd72df86aae5032f9c56a2b1ebdde14628453523601ac09eaa722ff0a7460 2024-04-05T03:11:38.481 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:38 smithi033 systemd[1]: Started Ceph osd.1 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:11:38.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:38 smithi151 ceph-mon[88672]: pgmap v153: 73 pgs: 2 active+undersized+degraded, 9 stale+active+clean, 2 active+undersized, 60 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:11:38.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:38 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2814731685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:38.980 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:38 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[114474]: 2024-04-05T03:11:38.712+0000 7f1b8e7f37c0 -1 Falling back to public interface 2024-04-05T03:11:39.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:39.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:39.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:39.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:39.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:39.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:39.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:40.645 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:40 smithi151 ceph-mon[88672]: pgmap v154: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:40.645 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:40 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2070871018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:40.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:40 smithi033 ceph-mon[101783]: pgmap v154: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:40.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:40 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2070871018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:40.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:40 smithi033 ceph-mon[97738]: pgmap v154: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:40.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:40 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2070871018' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:11:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:11:40] "GET /metrics HTTP/1.1" 200 34099 "" "Prometheus/2.43.0" 2024-04-05T03:11:41.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[114474]: 2024-04-05T03:11:41.466+0000 7f1b8e7f37c0 -1 osd.1 93 log_to_monitors true 2024-04-05T03:11:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: pgmap v155: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: from='osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:11:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:11:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/945980638' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: pgmap v155: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: from='osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/945980638' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:42.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: pgmap v155: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: from='osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:42.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:42 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/945980638' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:43.230 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:11:42 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[114474]: 2024-04-05T03:11:42.809+0000 7f1b7bc43700 -1 osd.1 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:11:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T03:11:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: failed to encode map e96 with expected crc 2024-04-05T03:11:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: failed to encode map e96 with expected crc 2024-04-05T03:11:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: osdmap e96: 8 total, 7 up, 8 in 2024-04-05T03:11:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: from='osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: from='osd.1 ' entity='osd.1' 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: failed to encode map e96 with expected crc 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: failed to encode map e96 with expected crc 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: osdmap e96: 8 total, 7 up, 8 in 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: from='osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: from='osd.1 ' entity='osd.1' 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:43.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:43 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: failed to encode map e96 with expected crc 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: failed to encode map e96 with expected crc 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: osdmap e96: 8 total, 7 up, 8 in 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: from='osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: from='osd.1 ' entity='osd.1' 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:43 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:44.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[97738]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:44.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[97738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:11:44.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[97738]: failed to encode map e97 with expected crc 2024-04-05T03:11:44.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[97738]: osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252] boot 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[97738]: osdmap e97: 8 total, 8 up, 8 in 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[97738]: failed to encode map e97 with expected crc 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[101783]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[101783]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[101783]: failed to encode map e97 with expected crc 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[101783]: osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252] boot 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[101783]: osdmap e97: 8 total, 8 up, 8 in 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:11:44.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:44 smithi033 ceph-mon[101783]: failed to encode map e97 with expected crc 2024-04-05T03:11:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:44 smithi151 ceph-mon[88672]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:11:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:44 smithi151 ceph-mon[88672]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:11:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:44 smithi151 ceph-mon[88672]: failed to encode map e97 with expected crc 2024-04-05T03:11:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:44 smithi151 ceph-mon[88672]: osd.1 [v2:172.21.15.33:6810/2680554252,v1:172.21.15.33:6811/2680554252] boot 2024-04-05T03:11:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:44 smithi151 ceph-mon[88672]: osdmap e97: 8 total, 8 up, 8 in 2024-04-05T03:11:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:44 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:11:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:44 smithi151 ceph-mon[88672]: failed to encode map e97 with expected crc 2024-04-05T03:11:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3450431278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: osdmap e98: 8 total, 8 up, 8 in 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3450431278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: osdmap e98: 8 total, 8 up, 8 in 2024-04-05T03:11:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:45 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3450431278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: osdmap e98: 8 total, 8 up, 8 in 2024-04-05T03:11:45.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:45.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:45 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:46.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:46 smithi033 ceph-mon[101783]: pgmap v159: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:46.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:46 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:46.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:46 smithi033 ceph-mon[101783]: failed to encode map e98 with expected crc 2024-04-05T03:11:46.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:46 smithi033 ceph-mon[97738]: pgmap v159: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:46.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:46 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:46.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:46 smithi033 ceph-mon[97738]: failed to encode map e98 with expected crc 2024-04-05T03:11:46.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:46 smithi151 ceph-mon[88672]: pgmap v159: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:46 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:46 smithi151 ceph-mon[88672]: failed to encode map e98 with expected crc 2024-04-05T03:11:46.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:11:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:11:46.745+0000 7f5169f0e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:11:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:11:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3949740829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:11:47.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:47 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3949740829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:11:47.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:11:47.779 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:47 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3949740829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[97738]: pgmap v161: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[97738]: pgmap v162: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[97738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:11:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[97738]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:11:48.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[101783]: pgmap v161: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[101783]: pgmap v162: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[101783]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:11:48.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:48 smithi033 ceph-mon[101783]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:11:48.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:48 smithi151 ceph-mon[88672]: pgmap v161: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:48.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:48 smithi151 ceph-mon[88672]: pgmap v162: 73 pgs: 3 peering, 4 active+undersized+degraded, 25 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:11:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:48 smithi151 ceph-mon[88672]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:11:48.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:48 smithi151 ceph-mon[88672]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:11:49.679 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:49 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3968836253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:49.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:49 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3968836253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:49.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:49 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3968836253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:50.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:50 smithi033 ceph-mon[101783]: pgmap v163: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:50.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:50 smithi033 ceph-mon[101783]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded) 2024-04-05T03:11:50.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:50 smithi033 ceph-mon[97738]: pgmap v163: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:50.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:50 smithi033 ceph-mon[97738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded) 2024-04-05T03:11:50.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:50 smithi151 ceph-mon[88672]: pgmap v163: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:50.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:50 smithi151 ceph-mon[88672]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded) 2024-04-05T03:11:51.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:11:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:11:50] "GET /metrics HTTP/1.1" 200 33949 "" "Prometheus/2.43.0" 2024-04-05T03:11:52.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:52 smithi033 ceph-mon[101783]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:52.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:52 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2251051102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:52.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:52 smithi033 ceph-mon[97738]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:52.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:52 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2251051102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:52.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:52 smithi151 ceph-mon[88672]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:52.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:52 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2251051102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:54.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:54 smithi033 ceph-mon[97738]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:54.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:54 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/181130444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:54.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:54 smithi033 ceph-mon[101783]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:54.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:54 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/181130444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:54 smithi151 ceph-mon[88672]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:54.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:54 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/181130444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:56.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:56 smithi033 ceph-mon[101783]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:56.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:56 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1875216285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:56 smithi033 ceph-mon[97738]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:56.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:56 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1875216285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:56.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:56 smithi151 ceph-mon[88672]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:56.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:56 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1875216285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:57.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:57 smithi033 ceph-mon[101783]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:57.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:57 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:57.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:57 smithi033 ceph-mon[97738]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:57.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:57 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:58.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:57 smithi151 ceph-mon[88672]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:58.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:57 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:11:58.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:58 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2658078723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:58.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:58 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2658078723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:59.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:58 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2658078723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:11:59.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:11:59 smithi033 ceph-mon[97738]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:59.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:11:59 smithi033 ceph-mon[101783]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:00.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:11:59 smithi151 ceph-mon[88672]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:01.135 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:12:00] "GET /metrics HTTP/1.1" 200 33949 "" "Prometheus/2.43.0" 2024-04-05T03:12:01.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[97738]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:01.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1622028689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:01.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:01.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:01.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[101783]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:01.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1622028689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:01.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:01.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:01 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:01 smithi151 ceph-mon[88672]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:02.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:01 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1622028689' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:02.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:01 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:02.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:01 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:02.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:12:02.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:02 smithi033 ceph-mon[97738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:12:02.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:02 smithi033 ceph-mon[97738]: Upgrade: osd.2 is safe to restart 2024-04-05T03:12:02.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:12:02.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:02 smithi033 ceph-mon[101783]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:12:02.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:02 smithi033 ceph-mon[101783]: Upgrade: osd.2 is safe to restart 2024-04-05T03:12:03.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:12:03.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:02 smithi151 ceph-mon[88672]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:12:03.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:02 smithi151 ceph-mon[88672]: Upgrade: osd.2 is safe to restart 2024-04-05T03:12:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[97738]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/939357687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[97738]: Upgrade: Updating osd.2 2024-04-05T03:12:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:12:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[97738]: Deploying daemon osd.2 on smithi033 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[101783]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/939357687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[101783]: Upgrade: Updating osd.2 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:03.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:03 smithi033 ceph-mon[101783]: Deploying daemon osd.2 on smithi033 2024-04-05T03:12:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:03 smithi151 ceph-mon[88672]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:03 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/939357687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:03 smithi151 ceph-mon[88672]: Upgrade: Updating osd.2 2024-04-05T03:12:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:03 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:03 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:12:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:03 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:04.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:03 smithi151 ceph-mon[88672]: Deploying daemon osd.2 on smithi033 2024-04-05T03:12:05.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:05 smithi033 ceph-mon[101783]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:05.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:05 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3864465845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:05.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:05 smithi033 ceph-mon[97738]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:05.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:05 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3864465845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:06.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:05 smithi151 ceph-mon[88672]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:06.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:05 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3864465845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:06.942 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:06 smithi033 systemd[1]: Stopping Ceph osd.2 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:12:07.230 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:06 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[55581]: 2024-04-05T03:12:06.942+0000 7f4b67cbe700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:12:07.230 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:06 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[55581]: 2024-04-05T03:12:06.942+0000 7f4b67cbe700 -1 osd.2 98 *** Got signal Terminated *** 2024-04-05T03:12:07.230 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:06 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[55581]: 2024-04-05T03:12:06.942+0000 7f4b67cbe700 -1 osd.2 98 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:12:07.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:07 smithi033 ceph-mon[97738]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:07.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:07 smithi033 ceph-mon[97738]: osd.2 marked itself down and dead 2024-04-05T03:12:07.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:07 smithi033 ceph-mon[101783]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:07.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:07 smithi033 ceph-mon[101783]: osd.2 marked itself down and dead 2024-04-05T03:12:08.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:07 smithi151 ceph-mon[88672]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:08.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:07 smithi151 ceph-mon[88672]: osd.2 marked itself down and dead 2024-04-05T03:12:08.542 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:08 smithi033 podman[118829]: 2024-04-05 03:12:08.248677279 +0000 UTC m=+1.422832527 container died 309a3fe45561fb9a93255ee146599b5740c506eb9a238cb14b41072ffe4776e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, RELEASE=quincy-6fda41f, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-04-05T03:12:08.910 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:08 smithi033 podman[118829]: 2024-04-05 03:12:08.651656576 +0000 UTC m=+1.825811818 container cleanup 309a3fe45561fb9a93255ee146599b5740c506eb9a238cb14b41072ffe4776e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-05T03:12:08.910 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:08 smithi033 bash[118829]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: osdmap e99: 8 total, 7 up, 8 in 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1268076952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.911 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: osdmap e99: 8 total, 7 up, 8 in 2024-04-05T03:12:08.912 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.912 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.912 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: failed to encode map e99 with expected crc 2024-04-05T03:12:08.912 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:08 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1268076952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: failed to encode map e99 with expected crc 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: failed to encode map e99 with expected crc 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: osdmap e99: 8 total, 7 up, 8 in 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: failed to encode map e99 with expected crc 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: failed to encode map e99 with expected crc 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: failed to encode map e99 with expected crc 2024-04-05T03:12:09.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:08 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1268076952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:09.230 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:08 smithi033 podman[118874]: 2024-04-05 03:12:08.911842161 +0000 UTC m=+0.662074636 container remove 309a3fe45561fb9a93255ee146599b5740c506eb9a238cb14b41072ffe4776e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-05T03:12:09.573 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:09 smithi033 podman[118966]: 2024-04-05 03:12:09.238150046 +0000 UTC m=+0.114139098 container create 95cc67112b9c9bb7091c426a155b459b57987fd22c879bda43ee79b73cf4b985 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-05T03:12:09.573 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:09 smithi033 podman[118966]: 2024-04-05 03:12:09.141874308 +0000 UTC m=+0.017863373 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:09.573 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:09 smithi033 podman[118966]: 2024-04-05 03:12:09.474517861 +0000 UTC m=+0.350506912 container init 95cc67112b9c9bb7091c426a155b459b57987fd22c879bda43ee79b73cf4b985 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:12:09.573 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:09 smithi033 podman[118966]: 2024-04-05 03:12:09.477849721 +0000 UTC m=+0.353838769 container start 95cc67112b9c9bb7091c426a155b459b57987fd22c879bda43ee79b73cf4b985 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-05T03:12:09.573 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:09 smithi033 podman[118966]: 2024-04-05 03:12:09.574354888 +0000 UTC m=+0.450343947 container attach 95cc67112b9c9bb7091c426a155b459b57987fd22c879bda43ee79b73cf4b985 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, GIT_CLEAN=True) 2024-04-05T03:12:09.980 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:09 smithi033 podman[118966]: 2024-04-05 03:12:09.772441179 +0000 UTC m=+0.648430237 container died 95cc67112b9c9bb7091c426a155b459b57987fd22c879bda43ee79b73cf4b985 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, RELEASE=squid-0028525, org.label-schema.license=GPLv2) 2024-04-05T03:12:09.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: pgmap v174: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:09.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: osdmap e100: 8 total, 7 up, 8 in 2024-04-05T03:12:09.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[101783]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: pgmap v174: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: osdmap e100: 8 total, 7 up, 8 in 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:09.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:09 smithi033 ceph-mon[97738]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: pgmap v174: 73 pgs: 2 stale+active+clean, 71 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: osdmap e100: 8 total, 7 up, 8 in 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:09 smithi151 ceph-mon[88672]: failed to encode map e100 with expected crc 2024-04-05T03:12:10.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:10 smithi033 podman[118991]: 2024-04-05 03:12:10.412207112 +0000 UTC m=+0.630620187 container remove 95cc67112b9c9bb7091c426a155b459b57987fd22c879bda43ee79b73cf4b985 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:12:10.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:10 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.2.service: Deactivated successfully. 2024-04-05T03:12:10.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:10 smithi033 systemd[1]: Stopped Ceph osd.2 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:12:10.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:10 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.2.service: Consumed 5.686s CPU time. 2024-04-05T03:12:10.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:10 smithi033 systemd[1]: Starting Ceph osd.2 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:12:11.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:10 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/777481544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:11.057 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:10 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/777481544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:11.058 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:10 smithi033 podman[119120]: 2024-04-05 03:12:10.83896194 +0000 UTC m=+0.105850109 container create 3c5a0ff0c1f5a73cc633de092ad6bc0cf899ba615ba10436a33bab81075d0c2c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-05T03:12:11.058 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:10 smithi033 podman[119120]: 2024-04-05 03:12:10.753792802 +0000 UTC m=+0.020680947 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:11.058 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:10 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/777481544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:11.058 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:12:10] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.43.0" 2024-04-05T03:12:11.480 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 podman[119120]: 2024-04-05 03:12:11.058388494 +0000 UTC m=+0.325276637 container init 3c5a0ff0c1f5a73cc633de092ad6bc0cf899ba615ba10436a33bab81075d0c2c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:12:11.480 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 podman[119120]: 2024-04-05 03:12:11.06243894 +0000 UTC m=+0.329327067 container start 3c5a0ff0c1f5a73cc633de092ad6bc0cf899ba615ba10436a33bab81075d0c2c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-05T03:12:11.480 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 podman[119120]: 2024-04-05 03:12:11.14993029 +0000 UTC m=+0.416818429 container attach 3c5a0ff0c1f5a73cc633de092ad6bc0cf899ba615ba10436a33bab81075d0c2c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-05T03:12:11.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-mon[97738]: pgmap v176: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+undersized, 69 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:11.861 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-mon[97738]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:12:11.861 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-mon[101783]: pgmap v176: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+undersized, 69 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:11.861 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-mon[101783]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:12:11.861 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate[119135]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:12:11.861 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate[119135]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-05T03:12:11.861 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 bash[119120]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 bash[119120]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 bash[119120]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 bash[119120]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 bash[119120]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 bash[119120]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 bash[119120]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate[119135]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate[119135]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate[119135]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate[119135]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate[119135]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-05T03:12:11.862 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:11 smithi033 podman[119120]: 2024-04-05 03:12:11.531719631 +0000 UTC m=+0.798607765 container died 3c5a0ff0c1f5a73cc633de092ad6bc0cf899ba615ba10436a33bab81075d0c2c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, ceph=True, RELEASE=squid-0028525, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-05T03:12:12.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:11 smithi151 ceph-mon[88672]: pgmap v176: 73 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 1 active+undersized, 69 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:12.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:11 smithi151 ceph-mon[88672]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:12:12.480 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 podman[119252]: 2024-04-05 03:12:12.204510731 +0000 UTC m=+0.667620363 container remove 3c5a0ff0c1f5a73cc633de092ad6bc0cf899ba615ba10436a33bab81075d0c2c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:12:12.480 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 podman[119340]: 2024-04-05 03:12:12.32914344 +0000 UTC m=+0.018307757 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:12.480 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 podman[119340]: 2024-04-05 03:12:12.430765168 +0000 UTC m=+0.119929476 container create cfa7f55e5875d8d4118b71ad51d32aa910445ea51c4b9649521174c51eedaf52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240403, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-05T03:12:12.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:12 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3969759935' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:12.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:12 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3969759935' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:12.981 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 podman[119340]: 2024-04-05 03:12:12.654918034 +0000 UTC m=+0.344082343 container init cfa7f55e5875d8d4118b71ad51d32aa910445ea51c4b9649521174c51eedaf52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-05T03:12:12.981 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 podman[119340]: 2024-04-05 03:12:12.658094156 +0000 UTC m=+0.347258464 container start cfa7f55e5875d8d4118b71ad51d32aa910445ea51c4b9649521174c51eedaf52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-05T03:12:12.981 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 ceph-osd[119362]: -- 172.21.15.33:0/237553391 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e2819bb680 con 0x55e281985400 2024-04-05T03:12:12.981 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 bash[119340]: cfa7f55e5875d8d4118b71ad51d32aa910445ea51c4b9649521174c51eedaf52 2024-04-05T03:12:12.981 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:12 smithi033 systemd[1]: Started Ceph osd.2 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:12:13.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:12 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3969759935' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:13.480 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[119358]: 2024-04-05T03:12:13.202+0000 7ffbcc0c37c0 -1 Falling back to public interface 2024-04-05T03:12:13.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[97738]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:13.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:13.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:13.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[101783]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:13.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:13 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:14.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:13 smithi151 ceph-mon[88672]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 6 active+undersized, 65 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:14.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:13 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:14.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:13 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:14.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:13 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:16.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:15 smithi033 ceph-mon[101783]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:16.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:15 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2515858524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:16.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:15 smithi033 ceph-mon[97738]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:16.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:15 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2515858524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:16.231 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:15 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[119358]: 2024-04-05T03:12:15.918+0000 7ffbcc0c37c0 -1 osd.2 98 log_to_monitors true 2024-04-05T03:12:16.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:15 smithi151 ceph-mon[88672]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:16.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:15 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2515858524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:16.980 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[119358]: 2024-04-05T03:12:16.839+0000 7ffbbd51b700 -1 osd.2 98 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:12:16.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[101783]: from='osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:12:16.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[101783]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:12:16.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:16.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[101783]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[97738]: from='osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[97738]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[97738]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:16.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:16 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:17.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:16 smithi151 ceph-mon[88672]: from='osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:12:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:16 smithi151 ceph-mon[88672]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:12:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:16 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:16 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:16 smithi151 ceph-mon[88672]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:16 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:17.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:16 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: from='osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: osdmap e101: 8 total, 7 up, 8 in 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3678577185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.220 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: from='osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: osdmap e101: 8 total, 7 up, 8 in 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3678577185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:18.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: from='osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: osdmap e101: 8 total, 7 up, 8 in 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3678577185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: failed to encode map e101 with expected crc 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:18.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: failed to encode map e101 with expected crc 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: failed to encode map e101 with expected crc 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: failed to encode map e102 with expected crc 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: failed to encode map e102 with expected crc 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840] boot 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: osdmap e102: 8 total, 8 up, 8 in 2024-04-05T03:12:19.179 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:12:19.180 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:18 smithi151 ceph-mon[88672]: osdmap e103: 8 total, 8 up, 8 in 2024-04-05T03:12:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: failed to encode map e101 with expected crc 2024-04-05T03:12:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: failed to encode map e101 with expected crc 2024-04-05T03:12:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: failed to encode map e102 with expected crc 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: failed to encode map e102 with expected crc 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840] boot 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: osdmap e102: 8 total, 8 up, 8 in 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[97738]: osdmap e103: 8 total, 8 up, 8 in 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: failed to encode map e101 with expected crc 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: failed to encode map e101 with expected crc 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: failed to encode map e102 with expected crc 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: failed to encode map e102 with expected crc 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: osd.2 [v2:172.21.15.33:6818/501263840,v1:172.21.15.33:6819/501263840] boot 2024-04-05T03:12:19.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: osdmap e102: 8 total, 8 up, 8 in 2024-04-05T03:12:19.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:12:19.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:18 smithi033 ceph-mon[101783]: osdmap e103: 8 total, 8 up, 8 in 2024-04-05T03:12:20.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:20.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2572265750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.231 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.232 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:19 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2572265750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:20.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: failed to encode map e103 with expected crc 2024-04-05T03:12:20.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:19 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2572265750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:20.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:12:20] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.43.0" 2024-04-05T03:12:21.480 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:21 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:12:21.278+0000 7f5169f0e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-05T03:12:22.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: pgmap v184: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: pgmap v185: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: pgmap v186: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3163209866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: pgmap v184: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: pgmap v185: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: pgmap v186: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:22.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:22.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-05T03:12:22.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:22.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.483 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:22 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3163209866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: pgmap v184: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: pgmap v185: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: pgmap v186: 73 pgs: 2 active+undersized+degraded, 15 active+undersized, 56 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:22.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:22.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:22.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-05T03:12:22.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:22.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:22.529 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:22 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3163209866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:23.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:23 smithi151 ceph-mon[88672]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-05T03:12:23.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:23 smithi033 ceph-mon[97738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-05T03:12:23.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:23 smithi033 ceph-mon[101783]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-05T03:12:24.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:24 smithi151 ceph-mon[88672]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:24.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:24 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1933237188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:24 smithi033 ceph-mon[97738]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:24 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1933237188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:24.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:24 smithi033 ceph-mon[101783]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:24.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:24 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1933237188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:26.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:26 smithi151 ceph-mon[88672]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:26 smithi033 ceph-mon[97738]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:26.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:26 smithi033 ceph-mon[101783]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:27.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:27 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2217030192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:27.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:27 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2217030192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:27.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:27 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2217030192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:28.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:28 smithi151 ceph-mon[88672]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:28.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:28 smithi033 ceph-mon[97738]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:28.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:28 smithi033 ceph-mon[101783]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:29.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:29 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2484864314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:29.631 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:29 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2484864314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:29.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:29 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2484864314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:30.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:30 smithi151 ceph-mon[88672]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:30 smithi033 ceph-mon[97738]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:30 smithi033 ceph-mon[101783]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:31.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:12:30] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.43.0" 2024-04-05T03:12:31.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:31 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1968243767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:31.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:31 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1968243767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:31.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:31 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1968243767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:32.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:32 smithi151 ceph-mon[88672]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:32.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:32.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[101783]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:32.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:32.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:32.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:32.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[97738]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:32.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:32.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:34.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:34 smithi151 ceph-mon[88672]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:34.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:34 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/338620735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:34.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:34 smithi033 ceph-mon[97738]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:34.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:34 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/338620735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:34 smithi033 ceph-mon[101783]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:34 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/338620735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:36 smithi151 ceph-mon[88672]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:36.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:36 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1367451726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:36.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:36 smithi033 ceph-mon[97738]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:36.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:36 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1367451726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:36.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:36 smithi033 ceph-mon[101783]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:36.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:36 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1367451726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:37.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:37 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:37.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:37 smithi033 ceph-mon[97738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:37.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:37 smithi033 ceph-mon[97738]: Upgrade: osd.3 is safe to restart 2024-04-05T03:12:37.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:37 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:37.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:37 smithi033 ceph-mon[101783]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:37.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:37 smithi033 ceph-mon[101783]: Upgrade: osd.3 is safe to restart 2024-04-05T03:12:37.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:37 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:37.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:37 smithi151 ceph-mon[88672]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:12:37.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:37 smithi151 ceph-mon[88672]: Upgrade: osd.3 is safe to restart 2024-04-05T03:12:38.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:38 smithi151 ceph-mon[88672]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:38.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:38 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:38.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:38 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:12:38.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:38 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:38.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:38 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/148427625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[101783]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/148427625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[97738]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:38.543 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:12:38.544 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:38.544 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:38 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/148427625' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:39.534 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:39 smithi033 ceph-mon[97738]: Upgrade: Updating osd.3 2024-04-05T03:12:39.534 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:39 smithi033 ceph-mon[97738]: Deploying daemon osd.3 on smithi033 2024-04-05T03:12:39.534 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:39 smithi033 ceph-mon[101783]: Upgrade: Updating osd.3 2024-04-05T03:12:39.534 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:39 smithi033 ceph-mon[101783]: Deploying daemon osd.3 on smithi033 2024-04-05T03:12:39.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:39 smithi151 ceph-mon[88672]: Upgrade: Updating osd.3 2024-04-05T03:12:39.787 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:39 smithi151 ceph-mon[88672]: Deploying daemon osd.3 on smithi033 2024-04-05T03:12:40.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:40 smithi033 ceph-mon[97738]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:40.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:40 smithi033 ceph-mon[101783]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:40.776 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:12:40] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.43.0" 2024-04-05T03:12:40.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:40 smithi151 ceph-mon[88672]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:41.389 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:41 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3444880965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:41.389 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:41 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3444880965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:41.389 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:41 smithi033 systemd[1]: Stopping Ceph osd.3 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:12:41.730 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[61729]: 2024-04-05T03:12:41.389+0000 7f8e4d823700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:12:41.730 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[61729]: 2024-04-05T03:12:41.389+0000 7f8e4d823700 -1 osd.3 103 *** Got signal Terminated *** 2024-04-05T03:12:41.730 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:41 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[61729]: 2024-04-05T03:12:41.389+0000 7f8e4d823700 -1 osd.3 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:12:41.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:41 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3444880965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:42.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:42 smithi033 ceph-mon[97738]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:42.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:42 smithi033 ceph-mon[97738]: osd.3 marked itself down and dead 2024-04-05T03:12:42.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:42 smithi033 ceph-mon[101783]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:42.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:42 smithi033 ceph-mon[101783]: osd.3 marked itself down and dead 2024-04-05T03:12:42.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:42 smithi151 ceph-mon[88672]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:42.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:42 smithi151 ceph-mon[88672]: osd.3 marked itself down and dead 2024-04-05T03:12:42.980 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:42 smithi033 podman[123704]: 2024-04-05 03:12:42.690251481 +0000 UTC m=+1.421416355 container died 2b3b4d99eb6569b93a7edcb0de80eeebbe2db97c7080d1e342a484bdeeef7216 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_CLEAN=True, RELEASE=quincy-6fda41f, ceph=True) 2024-04-05T03:12:43.292 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 podman[123704]: 2024-04-05 03:12:43.048252784 +0000 UTC m=+1.779417623 container cleanup 2b3b4d99eb6569b93a7edcb0de80eeebbe2db97c7080d1e342a484bdeeef7216 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-05T03:12:43.293 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 bash[123704]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3 2024-04-05T03:12:43.293 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 podman[123745]: 2024-04-05 03:12:43.224671691 +0000 UTC m=+0.527538471 container remove 2b3b4d99eb6569b93a7edcb0de80eeebbe2db97c7080d1e342a484bdeeef7216 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-05T03:12:43.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[101783]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:12:43.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[101783]: failed to encode map e104 with expected crc 2024-04-05T03:12:43.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[101783]: osdmap e104: 8 total, 7 up, 8 in 2024-04-05T03:12:43.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/744627568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:43.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[97738]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:12:43.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[97738]: failed to encode map e104 with expected crc 2024-04-05T03:12:43.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[97738]: osdmap e104: 8 total, 7 up, 8 in 2024-04-05T03:12:43.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:43 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/744627568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:43.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:43 smithi151 ceph-mon[88672]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:12:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:43 smithi151 ceph-mon[88672]: failed to encode map e104 with expected crc 2024-04-05T03:12:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:43 smithi151 ceph-mon[88672]: osdmap e104: 8 total, 7 up, 8 in 2024-04-05T03:12:43.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:43 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/744627568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:43.969 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 podman[123841]: 2024-04-05 03:12:43.553940051 +0000 UTC m=+0.018054100 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:43.969 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 podman[123841]: 2024-04-05 03:12:43.659121157 +0000 UTC m=+0.123235251 container create c74f396cfa49ecd246d1c545a9cda3f9a27e3b2663a8577ad97ea8731aaf1f24 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-0028525, GIT_BRANCH=HEAD) 2024-04-05T03:12:43.969 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 podman[123841]: 2024-04-05 03:12:43.88699622 +0000 UTC m=+0.351110270 container init c74f396cfa49ecd246d1c545a9cda3f9a27e3b2663a8577ad97ea8731aaf1f24 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:12:43.969 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 podman[123841]: 2024-04-05 03:12:43.890097518 +0000 UTC m=+0.354211568 container start c74f396cfa49ecd246d1c545a9cda3f9a27e3b2663a8577ad97ea8731aaf1f24 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:12:43.969 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:43 smithi033 podman[123841]: 2024-04-05 03:12:43.970424707 +0000 UTC m=+0.434538766 container attach c74f396cfa49ecd246d1c545a9cda3f9a27e3b2663a8577ad97ea8731aaf1f24 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_CLEAN=True, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:12:44.230 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:44 smithi033 podman[123841]: 2024-04-05 03:12:44.196140116 +0000 UTC m=+0.660254166 container died c74f396cfa49ecd246d1c545a9cda3f9a27e3b2663a8577ad97ea8731aaf1f24 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: pgmap v198: 73 pgs: 21 peering, 3 stale+active+clean, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: Health check failed: Reduced data availability: 5 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: osdmap e105: 8 total, 7 up, 8 in 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[97738]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: pgmap v198: 73 pgs: 21 peering, 3 stale+active+clean, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: Health check failed: Reduced data availability: 5 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: osdmap e105: 8 total, 7 up, 8 in 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.633 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:44 smithi033 ceph-mon[101783]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: pgmap v198: 73 pgs: 21 peering, 3 stale+active+clean, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: Health check failed: Reduced data availability: 5 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: osdmap e105: 8 total, 7 up, 8 in 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:44 smithi151 ceph-mon[88672]: failed to encode map e105 with expected crc 2024-04-05T03:12:44.884 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:44 smithi033 podman[123867]: 2024-04-05 03:12:44.866440466 +0000 UTC m=+0.664622347 container remove c74f396cfa49ecd246d1c545a9cda3f9a27e3b2663a8577ad97ea8731aaf1f24 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True) 2024-04-05T03:12:45.230 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:44 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.3.service: Deactivated successfully. 2024-04-05T03:12:45.230 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:44 smithi033 systemd[1]: Stopped Ceph osd.3 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:12:45.230 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:44 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.3.service: Consumed 5.361s CPU time. 2024-04-05T03:12:45.231 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:45 smithi033 systemd[1]: Starting Ceph osd.3 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:12:45.620 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:45 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2890538154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:45.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:45 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2890538154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:45.620 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:45 smithi033 podman[123996]: 2024-04-05 03:12:45.225345562 +0000 UTC m=+0.017856780 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:45.620 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:45 smithi033 podman[123996]: 2024-04-05 03:12:45.351253632 +0000 UTC m=+0.143764837 container create 85c05a527736b5dc9327aac2c9b922f7ffac143804ef3f4302ba68f5aafbc924 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate, RELEASE=squid-0028525, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-05T03:12:45.621 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:45 smithi033 podman[123996]: 2024-04-05 03:12:45.62153888 +0000 UTC m=+0.414050099 container init 85c05a527736b5dc9327aac2c9b922f7ffac143804ef3f4302ba68f5aafbc924 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403) 2024-04-05T03:12:45.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:45 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2890538154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:45.980 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:45 smithi033 podman[123996]: 2024-04-05 03:12:45.624836954 +0000 UTC m=+0.417348162 container start 85c05a527736b5dc9327aac2c9b922f7ffac143804ef3f4302ba68f5aafbc924 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-05T03:12:45.980 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:45 smithi033 podman[123996]: 2024-04-05 03:12:45.68737509 +0000 UTC m=+0.479886298 container attach 85c05a527736b5dc9327aac2c9b922f7ffac143804ef3f4302ba68f5aafbc924 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-04-05T03:12:46.382 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-mon[97738]: pgmap v200: 73 pgs: 1 active+undersized+degraded, 21 peering, 1 stale+active+clean, 6 active+undersized, 44 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-05T03:12:46.382 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-mon[97738]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:12:46.382 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-mon[101783]: pgmap v200: 73 pgs: 1 active+undersized+degraded, 21 peering, 1 stale+active+clean, 6 active+undersized, 44 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-05T03:12:46.382 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-mon[101783]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate[124013]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate[124013]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 bash[123996]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 bash[123996]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 bash[123996]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 bash[123996]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 bash[123996]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 bash[123996]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 bash[123996]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate[124013]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate[124013]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-05T03:12:46.383 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate[124013]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-05T03:12:46.384 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate[124013]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:12:46.384 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate[124013]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-05T03:12:46.384 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 podman[123996]: 2024-04-05 03:12:46.098043563 +0000 UTC m=+0.890554759 container died 85c05a527736b5dc9327aac2c9b922f7ffac143804ef3f4302ba68f5aafbc924 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:12:46.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:46 smithi151 ceph-mon[88672]: pgmap v200: 73 pgs: 1 active+undersized+degraded, 21 peering, 1 stale+active+clean, 6 active+undersized, 44 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-05T03:12:46.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:46 smithi151 ceph-mon[88672]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:12:46.980 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 podman[124146]: 2024-04-05 03:12:46.658308767 +0000 UTC m=+0.552190308 container remove 85c05a527736b5dc9327aac2c9b922f7ffac143804ef3f4302ba68f5aafbc924 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-0028525, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-05T03:12:46.980 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 podman[124211]: 2024-04-05 03:12:46.773652885 +0000 UTC m=+0.017884844 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:46.980 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:46 smithi033 podman[124211]: 2024-04-05 03:12:46.884508622 +0000 UTC m=+0.128740594 container create ca165607e43d885feda76099e5a64c37447195b253ba64614b0848fa29f0e2c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-05T03:12:47.480 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:47 smithi033 podman[124211]: 2024-04-05 03:12:47.137476673 +0000 UTC m=+0.381708605 container init ca165607e43d885feda76099e5a64c37447195b253ba64614b0848fa29f0e2c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-05T03:12:47.480 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:47 smithi033 podman[124211]: 2024-04-05 03:12:47.140618329 +0000 UTC m=+0.384850260 container start ca165607e43d885feda76099e5a64c37447195b253ba64614b0848fa29f0e2c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-0028525) 2024-04-05T03:12:47.480 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-osd[124230]: -- 172.21.15.33:0/1611691473 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559d65331680 con 0x559d652fac00 2024-04-05T03:12:47.480 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:47 smithi033 bash[124211]: ca165607e43d885feda76099e5a64c37447195b253ba64614b0848fa29f0e2c5 2024-04-05T03:12:47.481 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:47 smithi033 systemd[1]: Started Ceph osd.3 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:12:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[97738]: pgmap v201: 73 pgs: 1 active+undersized+degraded, 21 peering, 9 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-05T03:12:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1959082394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:47.981 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[124226]: 2024-04-05T03:12:47.679+0000 7fa4f27f97c0 -1 Falling back to public interface 2024-04-05T03:12:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[101783]: pgmap v201: 73 pgs: 1 active+undersized+degraded, 21 peering, 9 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-05T03:12:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:47 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1959082394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:48.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:47 smithi151 ceph-mon[88672]: pgmap v201: 73 pgs: 1 active+undersized+degraded, 21 peering, 9 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-05T03:12:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:48.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:47 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1959082394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:49.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:49 smithi151 ceph-mon[88672]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 10 pgs peering) 2024-04-05T03:12:49.662 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:49 smithi033 ceph-mon[101783]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 10 pgs peering) 2024-04-05T03:12:49.662 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:49 smithi033 ceph-mon[97738]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 10 pgs peering) 2024-04-05T03:12:50.509 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:50 smithi033 ceph-mon[97738]: pgmap v202: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:12:50.510 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:50 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1366763361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:50.510 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:50 smithi033 ceph-mon[101783]: pgmap v202: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:12:50.510 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:50 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1366763361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:50.510 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[124226]: 2024-04-05T03:12:50.414+0000 7fa4f27f97c0 -1 osd.3 103 log_to_monitors true 2024-04-05T03:12:50.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:50 smithi151 ceph-mon[88672]: pgmap v202: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:12:50.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:50 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1366763361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:50.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:12:50] "GET /metrics HTTP/1.1" 200 34228 "" "Prometheus/2.43.0" 2024-04-05T03:12:51.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:51 smithi151 ceph-mon[88672]: from='osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:12:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:51 smithi151 ceph-mon[88672]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:12:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:51 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:51 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:51 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:51 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[97738]: from='osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:12:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[97738]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:12:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[101783]: from='osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:12:51.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[101783]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:12:51.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:51.731 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:12:51 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[124226]: 2024-04-05T03:12:51.270+0000 7fa4e3c51700 -1 osd.3 103 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:12:52.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: pgmap v203: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: osdmap e106: 8 total, 7 up, 8 in 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: from='osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:52.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:52 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1152438797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:52.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: pgmap v203: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: osdmap e106: 8 total, 7 up, 8 in 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: from='osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1152438797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:52.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: pgmap v203: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: osdmap e106: 8 total, 7 up, 8 in 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: from='osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: failed to encode map e106 with expected crc 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:52.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:52 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1152438797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:53.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: failed to encode map e106 with expected crc 2024-04-05T03:12:53.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:12:53.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046] boot 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: osdmap e107: 8 total, 8 up, 8 in 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[97738]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: failed to encode map e106 with expected crc 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046] boot 2024-04-05T03:12:53.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: osdmap e107: 8 total, 8 up, 8 in 2024-04-05T03:12:53.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:12:53.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:53 smithi033 ceph-mon[101783]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: failed to encode map e106 with expected crc 2024-04-05T03:12:53.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:12:53.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: osd.3 [v2:172.21.15.33:6826/932773046,v1:172.21.15.33:6827/932773046] boot 2024-04-05T03:12:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: osdmap e107: 8 total, 8 up, 8 in 2024-04-05T03:12:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:12:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: failed to encode map e107 with expected crc 2024-04-05T03:12:53.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:53 smithi151 ceph-mon[88672]: failed to encode map e107 with expected crc 2024-04-05T03:12:54.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:54 smithi151 ceph-mon[88672]: pgmap v206: 73 pgs: 11 peering, 2 active+undersized+degraded, 18 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:54.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:54 smithi151 ceph-mon[88672]: osdmap e108: 8 total, 8 up, 8 in 2024-04-05T03:12:54.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:54 smithi151 ceph-mon[88672]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:54 smithi151 ceph-mon[88672]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:54 smithi151 ceph-mon[88672]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:54 smithi151 ceph-mon[88672]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[101783]: pgmap v206: 73 pgs: 11 peering, 2 active+undersized+degraded, 18 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:54.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[101783]: osdmap e108: 8 total, 8 up, 8 in 2024-04-05T03:12:54.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[101783]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[101783]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[101783]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[101783]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.581 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 4 2024-04-05T03:12:54.581 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[97738]: pgmap v206: 73 pgs: 11 peering, 2 active+undersized+degraded, 18 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:54.581 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[97738]: osdmap e108: 8 total, 8 up, 8 in 2024-04-05T03:12:54.581 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[97738]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.581 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[97738]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.581 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[97738]: failed to encode map e108 with expected crc 2024-04-05T03:12:54.581 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:54 smithi033 ceph-mon[97738]: failed to encode map e108 with expected crc 2024-04-05T03:12:55.245 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T03:12:55.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:55 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1891948624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:55.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:55 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1891948624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:55.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:55 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1891948624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:12:55.795 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:55.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:12:55 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:12:55.941+0000 7f5169f0e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-05T03:12:56.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: pgmap v208: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:56.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:56.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: pgmap v208: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:56.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:56.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:56 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:12:56.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: pgmap v208: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:12:56.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:56 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:12:57.336 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[97738]: pgmap v209: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:57.336 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[97738]: pgmap v210: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:57.336 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[97738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:12:57.336 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[97738]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-05T03:12:57.336 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[97738]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:57.336 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[101783]: pgmap v209: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:57.336 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[101783]: pgmap v210: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:57.337 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[101783]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:12:57.337 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[101783]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-05T03:12:57.337 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:57 smithi033 ceph-mon[101783]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:57.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:57 smithi151 ceph-mon[88672]: pgmap v209: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:57.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:57 smithi151 ceph-mon[88672]: pgmap v210: 73 pgs: 11 peering, 2 active+undersized+degraded, 13 active+undersized, 47 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-05T03:12:57.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:57 smithi151 ceph-mon[88672]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:12:57.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:57 smithi151 ceph-mon[88672]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-05T03:12:57.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:57 smithi151 ceph-mon[88672]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:12:57.674 INFO:teuthology.orchestra.run.smithi033.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:58.276 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:12:58.541 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:58.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[97738]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:12:58.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[97738]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:58.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[97738]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:58.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[101783]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:12:58.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[101783]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:58.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:58.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:58 smithi033 ceph-mon[101783]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:59.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:58 smithi151 ceph-mon[88672]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:12:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:58 smithi151 ceph-mon[88672]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:12:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:58 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:12:59.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:58 smithi151 ceph-mon[88672]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:12:59.878 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:12:59 smithi033 ceph-mon[97738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-04-05T03:12:59.878 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:12:59 smithi033 ceph-mon[101783]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-04-05T03:13:00.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:12:59 smithi151 ceph-mon[88672]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-04-05T03:13:00.333 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (7m) 9s ago 10m 26.5M - 0.25.0 c8568f914cd2 a9cf7c76c519 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi151 *:3000 error 9s ago 10m - - 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi033.rmzwzs smithi033 running (9m) 9s ago 9m 16.8M - 17.2.7-805-g6fda41ff a49113eacc58 c7d056f2104c 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi151.cxpowy smithi151 running (9m) 9s ago 9m 17.0M - 17.2.7-805-g6fda41ff a49113eacc58 0d112d294451 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi151 *:8443,9283,8765 running (6m) 9s ago 14m 452M - 19.0.0-2013-g0028525c d8c35eaf3c25 090d83bbd680 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (6m) 9s ago 16m 507M - 19.0.0-2013-g0028525c d8c35eaf3c25 a6e11bc7dc6c 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (5m) 9s ago 16m 49.0M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 7657e117c64f 2024-04-05T03:13:00.334 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi151 running (3m) 9s ago 15m 35.6M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 634be946ebb9 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (4m) 9s ago 15m 39.3M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 fb77798ae736 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (7m) 9s ago 10m 15.1M - 1.5.0 0da6a335fe13 6da1bdc228ef 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi151 *:9100 running (6m) 9s ago 10m 17.0M - 1.5.0 0da6a335fe13 18b78d165e69 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (2m) 9s ago 14m 49.2M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 6a19fd0140d9 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (82s) 9s ago 14m 49.3M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 22dcd72df86a 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (47s) 9s ago 13m 39.3M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 cfa7f55e5875 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (13s) 9s ago 13m 11.6M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 ca165607e43d 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi151 running (13m) 9s ago 13m 49.6M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b7f73016d7ed 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi151 running (12m) 9s ago 12m 51.9M 2480M 17.2.7-805-g6fda41ff a49113eacc58 e2607db1cdd0 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi151 running (12m) 9s ago 12m 51.6M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2063f47e19f2 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi151 running (11m) 9s ago 11m 53.4M 2480M 17.2.7-805-g6fda41ff a49113eacc58 164624ba7a6d 2024-04-05T03:13:00.335 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi151 *:9095 running (6m) 9s ago 11m 59.9M - 2.43.0 a07b618ecd1d c6c02f0b66d8 2024-04-05T03:13:00.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:00 smithi033 ceph-mon[97738]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:00.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:00 smithi033 ceph-mon[97738]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:13:00.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:13:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:13:00] "GET /metrics HTTP/1.1" 200 34228 "" "Prometheus/2.43.0" 2024-04-05T03:13:00.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:00 smithi033 ceph-mon[101783]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:00.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:00 smithi033 ceph-mon[101783]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:13:01.014 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:13:01.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:00 smithi151 ceph-mon[88672]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:01.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:00 smithi151 ceph-mon[88672]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:13:01.286 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:02.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:02.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:02.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:02 smithi033 ceph-mon[97738]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:02.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:02.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:02.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:02 smithi033 ceph-mon[101783]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:03.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:03.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:03.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:02 smithi151 ceph-mon[88672]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:03.150 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:13:03.150 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (7m) 12s ago 10m 26.5M - 0.25.0 c8568f914cd2 a9cf7c76c519 2024-04-05T03:13:03.150 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi151 *:3000 error 12s ago 10m - - 2024-04-05T03:13:03.150 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi033.rmzwzs smithi033 running (9m) 12s ago 9m 16.8M - 17.2.7-805-g6fda41ff a49113eacc58 c7d056f2104c 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foo.smithi151.cxpowy smithi151 running (9m) 12s ago 9m 17.0M - 17.2.7-805-g6fda41ff a49113eacc58 0d112d294451 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi151 *:8443,9283,8765 running (6m) 12s ago 15m 452M - 19.0.0-2013-g0028525c d8c35eaf3c25 090d83bbd680 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (6m) 12s ago 16m 507M - 19.0.0-2013-g0028525c d8c35eaf3c25 a6e11bc7dc6c 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (5m) 12s ago 16m 49.0M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 7657e117c64f 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi151 running (4m) 12s ago 15m 35.6M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 634be946ebb9 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (4m) 12s ago 15m 39.3M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 fb77798ae736 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (7m) 12s ago 10m 15.1M - 1.5.0 0da6a335fe13 6da1bdc228ef 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi151 *:9100 running (6m) 12s ago 10m 17.0M - 1.5.0 0da6a335fe13 18b78d165e69 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (2m) 12s ago 14m 49.2M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 6a19fd0140d9 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (85s) 12s ago 14m 49.3M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 22dcd72df86a 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (50s) 12s ago 13m 39.3M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 cfa7f55e5875 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (16s) 12s ago 13m 11.6M 2481M 19.0.0-2013-g0028525c d8c35eaf3c25 ca165607e43d 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi151 running (13m) 12s ago 13m 49.6M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b7f73016d7ed 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi151 running (12m) 12s ago 12m 51.9M 2480M 17.2.7-805-g6fda41ff a49113eacc58 e2607db1cdd0 2024-04-05T03:13:03.151 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi151 running (12m) 12s ago 12m 51.6M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2063f47e19f2 2024-04-05T03:13:03.152 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi151 running (11m) 12s ago 11m 53.4M 2480M 17.2.7-805-g6fda41ff a49113eacc58 164624ba7a6d 2024-04-05T03:13:03.152 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi151 *:9095 running (6m) 12s ago 11m 59.9M - 2.43.0 a07b618ecd1d c6c02f0b66d8 2024-04-05T03:13:03.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:03 smithi033 ceph-mon[97738]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:13:03.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:03 smithi033 ceph-mon[101783]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:13:03.757 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph versions' 2024-04-05T03:13:04.021 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:04.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:03 smithi151 ceph-mon[88672]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:13:04.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:04 smithi033 ceph-mon[101783]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:04.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:04 smithi033 ceph-mon[97738]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:05.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:04 smithi151 ceph-mon[88672]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:05.963 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 3 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 2 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)": 4, 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 4 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)": 2 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-04-05T03:13:05.964 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-04-05T03:13:05.965 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)": 6, 2024-04-05T03:13:05.965 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 9 2024-04-05T03:13:05.965 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-04-05T03:13:05.965 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-04-05T03:13:06.594 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:06.594 INFO:teuthology.run_tasks:Running task print... 2024-04-05T03:13:06.605 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-05T03:13:06.605 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-05T03:13:06.621 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-05T03:13:06.621 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-05T03:13:06.621 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-05T03:13:06.894 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:07.132 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:06 smithi033 ceph-mon[97738]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:07.132 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:06 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2717503061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:13:07.132 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:06 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:07.132 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:06 smithi033 ceph-mon[101783]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:07.132 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:06 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2717503061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:13:07.132 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:06 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:07.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:06 smithi151 ceph-mon[88672]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:07.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:06 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2717503061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:13:07.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:06 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:08.755 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:13:09.070 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:08 smithi033 ceph-mon[101783]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:09.070 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:08 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/4012836777' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:13:09.071 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:08 smithi033 ceph-mon[97738]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:09.071 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:08 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/4012836777' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:13:09.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:08 smithi151 ceph-mon[88672]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:09.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:08 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/4012836777' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:13:09.401 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":38,"flags":0,"active_gid":25934,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3635306300},{"type":"v1","addr":"172.21.15.33:6801","nonce":3635306300}]},"active_addr":"172.21.15.33:6801/3635306300","active_change":"2024-04-05T03:07:16.567702+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25579,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.33:8443/","prometheus":"http://172.21.15.33:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":88,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":1345740645}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3490829094}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":142782921}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":1911159284}]}]} 2024-04-05T03:13:09.405 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-05T03:13:09.405 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-05T03:13:09.405 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:13:09.665 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:10.973 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:10 smithi033 ceph-mon[97738]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:10.974 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:10 smithi033 ceph-mon[101783]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:10.974 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:13:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:13:10] "GET /metrics HTTP/1.1" 200 34222 "" "Prometheus/2.43.0" 2024-04-05T03:13:11.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:10 smithi151 ceph-mon[88672]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:11.459 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:13:11.459 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":108,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:12:53.254704+0000","last_up_change":"2024-04-05T03:12:52.246810+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T03:01:42.593775+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":84,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T03:02:47.380696+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T03:02:47.698394+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":92,"up_thru":107,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6803","nonce":2467730266}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6805","nonce":2467730266}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6809","nonce":2467730266}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6807","nonce":2467730266}]},"public_addr":"172.21.15.33:6803/2467730266","cluster_addr":"172.21.15.33:6805/2467730266","heartbeat_back_addr":"172.21.15.33:6809/2467730266","heartbeat_front_addr":"172.21.15.33:6807/2467730266","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":93,"up_from":97,"up_thru":107,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6811","nonce":2680554252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6813","nonce":2680554252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6817","nonce":2680554252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6815","nonce":2680554252}]},"public_addr":"172.21.15.33:6811/2680554252","cluster_addr":"172.21.15.33:6813/2680554252","heartbeat_back_addr":"172.21.15.33:6817/2680554252","heartbeat_front_addr":"172.21.15.33:6815/2680554252","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":98,"up_from":102,"up_thru":107,"down_at":99,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6819","nonce":501263840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6821","nonce":501263840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6825","nonce":501263840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6823","nonce":501263840}]},"public_addr":"172.21.15.33:6819/501263840","cluster_addr":"172.21.15.33:6821/501263840","heartbeat_back_addr":"172.21.15.33:6825/501263840","heartbeat_front_addr":"172.21.15.33:6823/501263840","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":103,"up_from":107,"up_thru":107,"down_at":104,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6827","nonce":932773046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6829","nonce":932773046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6833","nonce":932773046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6831","nonce":932773046}]},"public_addr":"172.21.15.33:6827/932773046","cluster_addr":"172.21.15.33:6829/932773046","heartbeat_back_addr":"172.21.15.33:6833/932773046","heartbeat_front_addr":"172.21.15.33:6831/932773046","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":93},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":98},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":103},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3372533218":"2024-04-06T03:07:16.567488+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.151:0/3319480227":"2024-04-06T03:06:30.789129+0000","172.21.15.151:0/2643836526":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/2236066571":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3439747338":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/2013404372":"2024-04-06T03:07:16.567488+0000","172.21.15.33:6800/2530405471":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/348031240":"2024-04-06T03:07:16.567488+0000","172.21.15.151:0/4038263137":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/1272921316":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/3239504883":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/2211132569":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/2530405471":"2024-04-06T03:05:25.920282+0000","172.21.15.33:6800/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/1920159030":"2024-04-06T03:05:25.920282+0000","172.21.15.151:0/2679395146":"2024-04-06T03:06:30.789129+0000","172.21.15.33:6801/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/327739392":"2024-04-06T03:07:16.567488+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.151:0/1876344250":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3979175263":"2024-04-06T03:07:16.567488+0000","172.21.15.33:0/3813037768":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/1632976587":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/1529339487":"2024-04-06T03:07:16.567488+0000","172.21.15.151:6834/1962390741":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/3009892262":"2024-04-06T03:07:16.567488+0000","172.21.15.151:0/2604554593":"2024-04-06T03:06:30.789129+0000","172.21.15.151:6835/1962390741":"2024-04-06T03:06:30.789129+0000","172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6800/327739392":"2024-04-06T03:07:16.567488+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:13:12.036 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.036 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:12.036 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:12.036 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.036 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.036 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:12.036 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1416890823' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:12.037 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:12.038 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:11 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1416890823' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.063 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-05T03:13:12.063 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:13:12.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:13:12.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:11 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1416890823' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:13:12.326 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:14.146 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:13:14.146 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":108,"fsid":"f4bb5524-f2f7-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:56:04.452263+0000","modified":"2024-04-05T03:12:53.254704+0000","last_up_change":"2024-04-05T03:12:52.246810+0000","last_in_change":"2024-04-05T03:00:54.326760+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:59:14.558935+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T03:01:42.593775+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":84,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T03:02:47.380696+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T03:02:47.698394+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"7102dab8-6fe6-43ae-b43a-8c063343eaf0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":92,"up_thru":107,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6803","nonce":2467730266}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6805","nonce":2467730266}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6809","nonce":2467730266}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":2467730266},{"type":"v1","addr":"172.21.15.33:6807","nonce":2467730266}]},"public_addr":"172.21.15.33:6803/2467730266","cluster_addr":"172.21.15.33:6805/2467730266","heartbeat_back_addr":"172.21.15.33:6809/2467730266","heartbeat_front_addr":"172.21.15.33:6807/2467730266","state":["exists","up"]},{"osd":1,"uuid":"046e9421-82b9-40de-b66d-958962a69c97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":93,"up_from":97,"up_thru":107,"down_at":94,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6811","nonce":2680554252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6813","nonce":2680554252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6817","nonce":2680554252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2680554252},{"type":"v1","addr":"172.21.15.33:6815","nonce":2680554252}]},"public_addr":"172.21.15.33:6811/2680554252","cluster_addr":"172.21.15.33:6813/2680554252","heartbeat_back_addr":"172.21.15.33:6817/2680554252","heartbeat_front_addr":"172.21.15.33:6815/2680554252","state":["exists","up"]},{"osd":2,"uuid":"cc74077e-8e35-4d06-ba7d-ad0b05e4e515","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":98,"up_from":102,"up_thru":107,"down_at":99,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6819","nonce":501263840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6821","nonce":501263840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6825","nonce":501263840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":501263840},{"type":"v1","addr":"172.21.15.33:6823","nonce":501263840}]},"public_addr":"172.21.15.33:6819/501263840","cluster_addr":"172.21.15.33:6821/501263840","heartbeat_back_addr":"172.21.15.33:6825/501263840","heartbeat_front_addr":"172.21.15.33:6823/501263840","state":["exists","up"]},{"osd":3,"uuid":"95b643ed-ea3a-4e88-a2bf-6cc4f398df73","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":103,"up_from":107,"up_thru":107,"down_at":104,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6827","nonce":932773046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6829","nonce":932773046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6833","nonce":932773046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":932773046},{"type":"v1","addr":"172.21.15.33:6831","nonce":932773046}]},"public_addr":"172.21.15.33:6827/932773046","cluster_addr":"172.21.15.33:6829/932773046","heartbeat_back_addr":"172.21.15.33:6833/932773046","heartbeat_front_addr":"172.21.15.33:6831/932773046","state":["exists","up"]},{"osd":4,"uuid":"70bbcc30-2695-4251-927c-63b0251eca78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6800","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6801","nonce":1041236039}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6802","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6803","nonce":1041236039}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6806","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6807","nonce":1041236039}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6804","nonce":1041236039},{"type":"v1","addr":"172.21.15.151:6805","nonce":1041236039}]},"public_addr":"172.21.15.151:6801/1041236039","cluster_addr":"172.21.15.151:6803/1041236039","heartbeat_back_addr":"172.21.15.151:6807/1041236039","heartbeat_front_addr":"172.21.15.151:6805/1041236039","state":["exists","up"]},{"osd":5,"uuid":"e7dc85ee-9174-4774-83fc-249a6f3b29d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6808","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6809","nonce":2180428192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6810","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6811","nonce":2180428192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6814","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6815","nonce":2180428192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6812","nonce":2180428192},{"type":"v1","addr":"172.21.15.151:6813","nonce":2180428192}]},"public_addr":"172.21.15.151:6809/2180428192","cluster_addr":"172.21.15.151:6811/2180428192","heartbeat_back_addr":"172.21.15.151:6815/2180428192","heartbeat_front_addr":"172.21.15.151:6813/2180428192","state":["exists","up"]},{"osd":6,"uuid":"b492b365-76da-4aae-a854-26d49a49b7b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6816","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6817","nonce":658137903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6818","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6819","nonce":658137903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6822","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6823","nonce":658137903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6820","nonce":658137903},{"type":"v1","addr":"172.21.15.151:6821","nonce":658137903}]},"public_addr":"172.21.15.151:6817/658137903","cluster_addr":"172.21.15.151:6819/658137903","heartbeat_back_addr":"172.21.15.151:6823/658137903","heartbeat_front_addr":"172.21.15.151:6821/658137903","state":["exists","up"]},{"osd":7,"uuid":"6a4bb8ad-d8b9-40cc-b066-429d1aba4543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6824","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6825","nonce":2209750861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6826","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6827","nonce":2209750861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6830","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6831","nonce":2209750861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.151:6828","nonce":2209750861},{"type":"v1","addr":"172.21.15.151:6829","nonce":2209750861}]},"public_addr":"172.21.15.151:6825/2209750861","cluster_addr":"172.21.15.151:6827/2209750861","heartbeat_back_addr":"172.21.15.151:6831/2209750861","heartbeat_front_addr":"172.21.15.151:6829/2209750861","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:23.392807+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:58:46.986407+0000","dead_epoch":93},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:11.190311+0000","dead_epoch":98},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:34.777409+0000","dead_epoch":103},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:59:56.797550+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:21.019426+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:00:45.165632+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T03:01:09.625253+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:6800/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.151:0/2643836526":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/1529339487":"2024-04-06T03:07:16.567488+0000","172.21.15.33:0/1632976587":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/412028809":"2024-04-06T02:57:02.231293+0000","172.21.15.151:0/4038263137":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/1272921316":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/3979175263":"2024-04-06T03:07:16.567488+0000","172.21.15.33:0/3937065412":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/4010230297":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/3239504883":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6801/377470712":"2024-04-06T02:57:02.231293+0000","172.21.15.33:0/1920159030":"2024-04-06T03:05:25.920282+0000","172.21.15.33:6801/327739392":"2024-04-06T03:07:16.567488+0000","172.21.15.151:0/2679395146":"2024-04-06T03:06:30.789129+0000","172.21.15.33:6801/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:0/3439747338":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/2236066571":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/1861802444":"2024-04-06T02:56:38.541458+0000","172.21.15.33:6800/1252195287":"2024-04-06T03:01:44.019614+0000","172.21.15.33:6800/327739392":"2024-04-06T03:07:16.567488+0000","172.21.15.33:6801/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/1700247573":"2024-04-06T02:57:02.231293+0000","172.21.15.33:6801/2530405471":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/2211132569":"2024-04-06T03:01:44.019614+0000","172.21.15.151:0/3319480227":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/3009892262":"2024-04-06T03:07:16.567488+0000","172.21.15.151:6834/1962390741":"2024-04-06T03:06:30.789129+0000","172.21.15.33:6800/1767611051":"2024-04-06T02:56:38.541458+0000","172.21.15.33:0/3508628663":"2024-04-06T02:56:38.541458+0000","172.21.15.151:0/1876344250":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/3813037768":"2024-04-06T03:05:25.920282+0000","172.21.15.33:6800/2530405471":"2024-04-06T03:05:25.920282+0000","172.21.15.33:0/348031240":"2024-04-06T03:07:16.567488+0000","172.21.15.33:0/2013404372":"2024-04-06T03:07:16.567488+0000","172.21.15.151:0/2604554593":"2024-04-06T03:06:30.789129+0000","172.21.15.151:6835/1962390741":"2024-04-06T03:06:30.789129+0000","172.21.15.33:0/3372533218":"2024-04-06T03:07:16.567488+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:13:14.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:14 smithi033 ceph-mon[97738]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:14.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:14 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2248364038' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:13:14.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:14 smithi033 ceph-mon[101783]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:14.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:14 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2248364038' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:13:14.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:14 smithi151 ceph-mon[88672]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:14.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:14 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2248364038' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:13:14.831 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-05T03:13:14.831 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-05T03:13:14.831 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-05T03:13:14.832 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-05T03:13:14.832 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-05T03:13:14.833 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-05T03:13:14.833 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-05T03:13:14.833 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-05T03:13:15.373 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:15.379 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:15.380 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:15.383 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:15.383 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:15.386 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:15.387 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:15.390 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:16.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:16 smithi033 ceph-mon[97738]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:16.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:16 smithi033 ceph-mon[101783]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:16.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:16 smithi151 ceph-mon[88672]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:17.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:17.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:18.310 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:18 smithi033 ceph-mon[101783]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:18.310 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:18 smithi033 ceph-mon[97738]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:18.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:18 smithi151 ceph-mon[88672]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:20.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:20 smithi033 ceph-mon[97738]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:20.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:20 smithi033 ceph-mon[101783]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:20.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:20 smithi151 ceph-mon[88672]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:21.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:13:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:13:20] "GET /metrics HTTP/1.1" 200 34215 "" "Prometheus/2.43.0" 2024-04-05T03:13:22.355 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:22 smithi033 ceph-mon[101783]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:22.355 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:22 smithi033 ceph-mon[97738]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:22.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:22 smithi151 ceph-mon[88672]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:24.323 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:24 smithi033 ceph-mon[97738]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:24.323 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:24 smithi033 ceph-mon[101783]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:24.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:24 smithi151 ceph-mon[88672]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:25.692 INFO:teuthology.orchestra.run.smithi033.stdout:459561500680 2024-04-05T03:13:25.692 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-05T03:13:26.463 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:26 smithi033 ceph-mon[97738]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:26.463 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:26 smithi033 ceph-mon[101783]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:26.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:26 smithi151 ceph-mon[88672]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:27.560 INFO:teuthology.orchestra.run.smithi033.stdout:167503724703 2024-04-05T03:13:27.560 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-05T03:13:27.957 INFO:teuthology.orchestra.run.smithi033.stdout:201863463067 2024-04-05T03:13:27.957 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-05T03:13:28.453 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:28 smithi033 ceph-mon[97738]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:28.453 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:28 smithi033 ceph-mon[101783]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:28.476 INFO:teuthology.orchestra.run.smithi033.stdout:133143986340 2024-04-05T03:13:28.476 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-05T03:13:28.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:28 smithi151 ceph-mon[88672]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:29.526 INFO:teuthology.orchestra.run.smithi033.stdout:236223201430 2024-04-05T03:13:29.526 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-05T03:13:30.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:30 smithi033 ceph-mon[97738]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:30.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:30 smithi033 ceph-mon[101783]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:30.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:30 smithi151 ceph-mon[88672]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:30.810 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:13:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:13:30] "GET /metrics HTTP/1.1" 200 34215 "" "Prometheus/2.43.0" 2024-04-05T03:13:31.515 INFO:teuthology.orchestra.run.smithi033.stdout:438086664208 2024-04-05T03:13:31.516 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-05T03:13:31.616 INFO:teuthology.orchestra.run.smithi033.stdout:416611827735 2024-04-05T03:13:31.616 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-05T03:13:32.076 INFO:teuthology.orchestra.run.smithi033.stdout:395136991269 2024-04-05T03:13:32.076 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-05T03:13:32.121 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:32.121 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:32.133 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:32.412 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:32.414 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:32.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:32 smithi033 ceph-mon[97738]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:32.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:32.426 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:32 smithi033 ceph-mon[101783]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:32.426 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:32.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:32 smithi151 ceph-mon[88672]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:32.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:34.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:34 smithi033 ceph-mon[97738]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:34.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:34 smithi033 ceph-mon[101783]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:34.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:34 smithi151 ceph-mon[88672]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:36.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:36 smithi033 ceph-mon[97738]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:36.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:36 smithi033 ceph-mon[101783]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:36.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:36 smithi151 ceph-mon[88672]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:37.382 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:38.035 INFO:teuthology.orchestra.run.smithi033.stdout:167503724705 2024-04-05T03:13:38.143 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:38.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:38 smithi033 ceph-mon[97738]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:38.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:38 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3645400813' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:13:38.424 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:38 smithi033 ceph-mon[101783]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:38.425 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:38 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3645400813' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:13:38.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:38 smithi151 ceph-mon[88672]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:38.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:38 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3645400813' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:13:38.866 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:38.993 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724703 got 167503724705 for osd.5 2024-04-05T03:13:38.993 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:39.117 INFO:teuthology.orchestra.run.smithi033.stdout:459561500683 2024-04-05T03:13:39.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:39 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2371247237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:13:39.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:39 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2371247237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:13:39.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:39 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2371247237' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:13:39.791 INFO:tasks.cephadm.ceph_manager.ceph:need seq 459561500680 got 459561500683 for osd.3 2024-04-05T03:13:39.791 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:40.318 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:40 smithi033 ceph-mon[101783]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:40.319 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:40 smithi033 ceph-mon[97738]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:40.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:40 smithi151 ceph-mon[88672]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:13:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:13:40] "GET /metrics HTTP/1.1" 200 34222 "" "Prometheus/2.43.0" 2024-04-05T03:13:42.164 INFO:teuthology.orchestra.run.smithi033.stdout:236223201433 2024-04-05T03:13:42.460 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:42 smithi033 ceph-mon[101783]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:42.461 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:42 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1061574938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:13:42.461 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:42 smithi033 ceph-mon[97738]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:42.461 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:42 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1061574938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:13:42.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:42 smithi151 ceph-mon[88672]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:42.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:42 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1061574938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:13:42.775 INFO:teuthology.orchestra.run.smithi033.stdout:201863463070 2024-04-05T03:13:43.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:43 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3300806757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:13:43.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:43 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3300806757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:13:43.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:43 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3300806757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:13:43.990 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201430 got 236223201433 for osd.7 2024-04-05T03:13:43.990 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:44.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:44 smithi033 ceph-mon[97738]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:44.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:44 smithi033 ceph-mon[101783]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:44.331 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463067 got 201863463070 for osd.6 2024-04-05T03:13:44.331 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:44.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:44 smithi151 ceph-mon[88672]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:45.362 INFO:teuthology.orchestra.run.smithi033.stdout:438086664211 2024-04-05T03:13:46.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 438086664208 got 438086664211 for osd.2 2024-04-05T03:13:46.343 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:46.458 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:46 smithi033 ceph-mon[97738]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:46.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:46 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3261926273' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:13:46.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:46 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1002817538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:13:46.459 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:46 smithi033 ceph-mon[101783]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:46.459 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:46 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3261926273' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:13:46.459 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:46 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1002817538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:13:46.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:46 smithi151 ceph-mon[88672]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:46.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:46 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3261926273' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:13:46.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:46 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1002817538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:13:46.660 INFO:teuthology.orchestra.run.smithi033.stdout:133143986344 2024-04-05T03:13:47.236 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986340 got 133143986344 for osd.4 2024-04-05T03:13:47.236 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:47.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:47.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:47.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:47.659 INFO:teuthology.orchestra.run.smithi033.stdout:416611827738 2024-04-05T03:13:48.047 INFO:teuthology.orchestra.run.smithi033.stdout:395136991273 2024-04-05T03:13:48.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:48 smithi033 ceph-mon[97738]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:48.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:48 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/632738430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:13:48.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:48 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2086673305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:13:48.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:48 smithi033 ceph-mon[101783]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:48.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:48 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/632738430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:13:48.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:48 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2086673305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:13:48.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:48 smithi151 ceph-mon[88672]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:48 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/632738430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:13:48.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:48 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2086673305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:13:48.547 INFO:tasks.cephadm.ceph_manager.ceph:need seq 416611827735 got 416611827738 for osd.1 2024-04-05T03:13:48.547 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:49.239 INFO:tasks.cephadm.ceph_manager.ceph:need seq 395136991269 got 395136991273 for osd.0 2024-04-05T03:13:49.240 DEBUG:teuthology.parallel:result is None 2024-04-05T03:13:49.240 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-05T03:13:49.240 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:13:49.509 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:50.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:50 smithi033 ceph-mon[97738]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:50.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:50 smithi033 ceph-mon[101783]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:50.530 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:50 smithi151 ceph-mon[88672]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:50.911 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:13:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:13:50] "GET /metrics HTTP/1.1" 200 34221 "" "Prometheus/2.43.0" 2024-04-05T03:13:51.346 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:13:51.347 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:13:52.072 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":241,"stamp":"2024-04-05T03:13:51.106132+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":13916,"num_write":1146,"num_write_kb":15031,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1338940,"kb_used_data":22848,"kb_used_omap":36,"kb_used_meta":1315931,"kb_avail":748622276,"statfs":{"total":767960285184,"available":766589210624,"internally_reserved":0,"allocated":23396352,"data_stored":19186071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37083,"internal_metadata":1347514149},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002686"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265583+0000","last_change":"2024-04-05T03:11:44.826962+0000","last_active":"2024-04-05T03:12:52.265583+0000","last_peered":"2024-04-05T03:12:52.265583+0000","last_clean":"2024-04-05T03:12:52.265583+0000","last_became_active":"2024-04-05T03:11:44.826572+0000","last_became_peered":"2024-04-05T03:11:44.826572+0000","last_unstale":"2024-04-05T03:12:52.265583+0000","last_undegraded":"2024-04-05T03:12:52.265583+0000","last_fullsized":"2024-04-05T03:12:52.265583+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:28:53.091252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272871+0000","last_change":"2024-04-05T03:12:18.831902+0000","last_active":"2024-04-05T03:12:53.272871+0000","last_peered":"2024-04-05T03:12:53.272871+0000","last_clean":"2024-04-05T03:12:53.272871+0000","last_became_active":"2024-04-05T03:12:18.831510+0000","last_became_peered":"2024-04-05T03:12:18.831510+0000","last_unstale":"2024-04-05T03:12:53.272871+0000","last_undegraded":"2024-04-05T03:12:53.272871+0000","last_fullsized":"2024-04-05T03:12:53.272871+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:41:57.365084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.287316+0000","last_change":"2024-04-05T03:12:53.287316+0000","last_active":"2024-04-05T03:12:53.287316+0000","last_peered":"2024-04-05T03:12:53.287316+0000","last_clean":"2024-04-05T03:12:53.287316+0000","last_became_active":"2024-04-05T03:12:53.286740+0000","last_became_peered":"2024-04-05T03:12:53.286740+0000","last_unstale":"2024-04-05T03:12:53.287316+0000","last_undegraded":"2024-04-05T03:12:53.287316+0000","last_fullsized":"2024-04-05T03:12:53.287316+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:31:51.207997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":77,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.285988+0000","last_change":"2024-04-05T03:12:53.285988+0000","last_active":"2024-04-05T03:12:53.285988+0000","last_peered":"2024-04-05T03:12:53.285988+0000","last_clean":"2024-04-05T03:12:53.285988+0000","last_became_active":"2024-04-05T03:12:53.285634+0000","last_became_peered":"2024-04-05T03:12:53.285634+0000","last_unstale":"2024-04-05T03:12:53.285988+0000","last_undegraded":"2024-04-05T03:12:53.285988+0000","last_fullsized":"2024-04-05T03:12:53.285988+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:00:45.210744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283853+0000","last_change":"2024-04-05T03:12:53.283853+0000","last_active":"2024-04-05T03:12:53.283853+0000","last_peered":"2024-04-05T03:12:53.283853+0000","last_clean":"2024-04-05T03:12:53.283853+0000","last_became_active":"2024-04-05T03:12:53.283470+0000","last_became_peered":"2024-04-05T03:12:53.283470+0000","last_unstale":"2024-04-05T03:12:53.283853+0000","last_undegraded":"2024-04-05T03:12:53.283853+0000","last_fullsized":"2024-04-05T03:12:53.283853+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:07:07.647098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.260283+0000","last_change":"2024-04-05T03:02:48.458396+0000","last_active":"2024-04-05T03:12:52.260283+0000","last_peered":"2024-04-05T03:12:52.260283+0000","last_clean":"2024-04-05T03:12:52.260283+0000","last_became_active":"2024-04-05T03:02:48.458092+0000","last_became_peered":"2024-04-05T03:02:48.458092+0000","last_unstale":"2024-04-05T03:12:52.260283+0000","last_undegraded":"2024-04-05T03:12:52.260283+0000","last_fullsized":"2024-04-05T03:12:52.260283+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:43.973088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283628+0000","last_change":"2024-04-05T03:12:53.283628+0000","last_active":"2024-04-05T03:12:53.283628+0000","last_peered":"2024-04-05T03:12:53.283628+0000","last_clean":"2024-04-05T03:12:53.283628+0000","last_became_active":"2024-04-05T03:12:53.283299+0000","last_became_peered":"2024-04-05T03:12:53.283299+0000","last_unstale":"2024-04-05T03:12:53.283628+0000","last_undegraded":"2024-04-05T03:12:53.283628+0000","last_fullsized":"2024-04-05T03:12:53.283628+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:16:07.905585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":71,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258823+0000","last_change":"2024-04-05T03:11:44.823272+0000","last_active":"2024-04-05T03:12:52.258823+0000","last_peered":"2024-04-05T03:12:52.258823+0000","last_clean":"2024-04-05T03:12:52.258823+0000","last_became_active":"2024-04-05T03:11:44.822883+0000","last_became_peered":"2024-04-05T03:11:44.822883+0000","last_unstale":"2024-04-05T03:12:52.258823+0000","last_undegraded":"2024-04-05T03:12:52.258823+0000","last_fullsized":"2024-04-05T03:12:52.258823+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:22:55.498909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283492+0000","last_change":"2024-04-05T03:12:53.283492+0000","last_active":"2024-04-05T03:12:53.283492+0000","last_peered":"2024-04-05T03:12:53.283492+0000","last_clean":"2024-04-05T03:12:53.283492+0000","last_became_active":"2024-04-05T03:12:53.283178+0000","last_became_peered":"2024-04-05T03:12:53.283178+0000","last_unstale":"2024-04-05T03:12:53.283492+0000","last_undegraded":"2024-04-05T03:12:53.283492+0000","last_fullsized":"2024-04-05T03:12:53.283492+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:36:55.383542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":57,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272082+0000","last_change":"2024-04-05T03:10:31.867161+0000","last_active":"2024-04-05T03:12:53.272082+0000","last_peered":"2024-04-05T03:12:53.272082+0000","last_clean":"2024-04-05T03:12:53.272082+0000","last_became_active":"2024-04-05T03:10:31.866213+0000","last_became_peered":"2024-04-05T03:10:31.866213+0000","last_unstale":"2024-04-05T03:12:53.272082+0000","last_undegraded":"2024-04-05T03:12:53.272082+0000","last_fullsized":"2024-04-05T03:12:53.272082+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:09:11.571255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266774+0000","last_change":"2024-04-05T03:02:48.902933+0000","last_active":"2024-04-05T03:12:52.266774+0000","last_peered":"2024-04-05T03:12:52.266774+0000","last_clean":"2024-04-05T03:12:52.266774+0000","last_became_active":"2024-04-05T03:02:48.902583+0000","last_became_peered":"2024-04-05T03:02:48.902583+0000","last_unstale":"2024-04-05T03:12:52.266774+0000","last_undegraded":"2024-04-05T03:12:52.266774+0000","last_fullsized":"2024-04-05T03:12:52.266774+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:11:03.431547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272740+0000","last_change":"2024-04-05T03:11:44.824952+0000","last_active":"2024-04-05T03:12:53.272740+0000","last_peered":"2024-04-05T03:12:53.272740+0000","last_clean":"2024-04-05T03:12:53.272740+0000","last_became_active":"2024-04-05T03:11:44.824187+0000","last_became_peered":"2024-04-05T03:11:44.824187+0000","last_unstale":"2024-04-05T03:12:53.272740+0000","last_undegraded":"2024-04-05T03:12:53.272740+0000","last_fullsized":"2024-04-05T03:12:53.272740+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:39:01.880319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'50","reported_seq":175,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267530+0000","last_change":"2024-04-05T03:12:18.835306+0000","last_active":"2024-04-05T03:12:52.267530+0000","last_peered":"2024-04-05T03:12:52.267530+0000","last_clean":"2024-04-05T03:12:52.267530+0000","last_became_active":"2024-04-05T03:12:18.834232+0000","last_became_peered":"2024-04-05T03:12:18.834232+0000","last_unstale":"2024-04-05T03:12:52.267530+0000","last_undegraded":"2024-04-05T03:12:52.267530+0000","last_fullsized":"2024-04-05T03:12:52.267530+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:40:27.851343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036085400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":101,"num_read_kb":1566,"num_write":53,"num_write_kb":1295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286266+0000","last_change":"2024-04-05T03:12:53.286266+0000","last_active":"2024-04-05T03:12:53.286266+0000","last_peered":"2024-04-05T03:12:53.286266+0000","last_clean":"2024-04-05T03:12:53.286266+0000","last_became_active":"2024-04-05T03:12:53.285928+0000","last_became_peered":"2024-04-05T03:12:53.285928+0000","last_unstale":"2024-04-05T03:12:53.286266+0000","last_undegraded":"2024-04-05T03:12:53.286266+0000","last_fullsized":"2024-04-05T03:12:53.286266+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:46:04.658838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'209","reported_seq":794,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272110+0000","last_change":"2024-04-05T03:11:44.824531+0000","last_active":"2024-04-05T03:12:53.272110+0000","last_peered":"2024-04-05T03:12:53.272110+0000","last_clean":"2024-04-05T03:12:53.272110+0000","last_became_active":"2024-04-05T03:11:44.823951+0000","last_became_peered":"2024-04-05T03:11:44.823951+0000","last_unstale":"2024-04-05T03:12:53.272110+0000","last_undegraded":"2024-04-05T03:12:53.272110+0000","last_fullsized":"2024-04-05T03:12:53.272110+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":209,"log_dups_size":0,"ondisk_log_size":209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:51.684899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074036299999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1296,"num_read_kb":1383,"num_write":166,"num_write_kb":238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286319+0000","last_change":"2024-04-05T03:12:53.286319+0000","last_active":"2024-04-05T03:12:53.286319+0000","last_peered":"2024-04-05T03:12:53.286319+0000","last_clean":"2024-04-05T03:12:53.286319+0000","last_became_active":"2024-04-05T03:12:53.285932+0000","last_became_peered":"2024-04-05T03:12:53.285932+0000","last_unstale":"2024-04-05T03:12:53.286319+0000","last_undegraded":"2024-04-05T03:12:53.286319+0000","last_fullsized":"2024-04-05T03:12:53.286319+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:48:06.327074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271730+0000","last_change":"2024-04-05T03:10:31.866556+0000","last_active":"2024-04-05T03:12:53.271730+0000","last_peered":"2024-04-05T03:12:53.271730+0000","last_clean":"2024-04-05T03:12:53.271730+0000","last_became_active":"2024-04-05T03:10:31.865371+0000","last_became_peered":"2024-04-05T03:10:31.865371+0000","last_unstale":"2024-04-05T03:12:53.271730+0000","last_undegraded":"2024-04-05T03:12:53.271730+0000","last_fullsized":"2024-04-05T03:12:53.271730+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:48:34.182823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272547+0000","last_change":"2024-04-05T03:12:18.834658+0000","last_active":"2024-04-05T03:12:53.272547+0000","last_peered":"2024-04-05T03:12:53.272547+0000","last_clean":"2024-04-05T03:12:53.272547+0000","last_became_active":"2024-04-05T03:12:18.834340+0000","last_became_peered":"2024-04-05T03:12:18.834340+0000","last_unstale":"2024-04-05T03:12:53.272547+0000","last_undegraded":"2024-04-05T03:12:53.272547+0000","last_fullsized":"2024-04-05T03:12:53.272547+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:56:47.305361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"84'67","reported_seq":325,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.264221+0000","last_change":"2024-04-05T03:10:31.863085+0000","last_active":"2024-04-05T03:12:52.264221+0000","last_peered":"2024-04-05T03:12:52.264221+0000","last_clean":"2024-04-05T03:12:52.264221+0000","last_became_active":"2024-04-05T03:10:31.862321+0000","last_became_peered":"2024-04-05T03:10:31.862321+0000","last_unstale":"2024-04-05T03:12:52.264221+0000","last_undegraded":"2024-04-05T03:12:52.264221+0000","last_fullsized":"2024-04-05T03:12:52.264221+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:27:38.911052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028041299999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":410,"num_read_kb":2404,"num_write":65,"num_write_kb":6191,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272882+0000","last_change":"2024-04-05T03:12:18.831144+0000","last_active":"2024-04-05T03:12:53.272882+0000","last_peered":"2024-04-05T03:12:53.272882+0000","last_clean":"2024-04-05T03:12:53.272882+0000","last_became_active":"2024-04-05T03:12:18.830791+0000","last_became_peered":"2024-04-05T03:12:18.830791+0000","last_unstale":"2024-04-05T03:12:53.272882+0000","last_undegraded":"2024-04-05T03:12:53.272882+0000","last_fullsized":"2024-04-05T03:12:53.272882+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:54:42.730284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281338+0000","last_change":"2024-04-05T03:12:53.281338+0000","last_active":"2024-04-05T03:12:53.281338+0000","last_peered":"2024-04-05T03:12:53.281338+0000","last_clean":"2024-04-05T03:12:53.281338+0000","last_became_active":"2024-04-05T03:12:53.281008+0000","last_became_peered":"2024-04-05T03:12:53.281008+0000","last_unstale":"2024-04-05T03:12:53.281338+0000","last_undegraded":"2024-04-05T03:12:53.281338+0000","last_fullsized":"2024-04-05T03:12:53.281338+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:10:52.739314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'43","reported_seq":299,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266213+0000","last_change":"2024-04-05T03:11:44.821082+0000","last_active":"2024-04-05T03:12:52.266213+0000","last_peered":"2024-04-05T03:12:52.266213+0000","last_clean":"2024-04-05T03:12:52.266213+0000","last_became_active":"2024-04-05T03:11:44.820671+0000","last_became_peered":"2024-04-05T03:11:44.820671+0000","last_unstale":"2024-04-05T03:12:52.266213+0000","last_undegraded":"2024-04-05T03:12:52.266213+0000","last_fullsized":"2024-04-05T03:12:52.266213+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":43,"log_dups_size":0,"ondisk_log_size":43,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:40:01.852471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034224499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":329,"num_read_kb":305,"num_write":22,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.262555+0000","last_change":"2024-04-05T03:02:50.003073+0000","last_active":"2024-04-05T03:12:52.262555+0000","last_peered":"2024-04-05T03:12:52.262555+0000","last_clean":"2024-04-05T03:12:52.262555+0000","last_became_active":"2024-04-05T03:02:50.002726+0000","last_became_peered":"2024-04-05T03:02:50.002726+0000","last_unstale":"2024-04-05T03:12:52.262555+0000","last_undegraded":"2024-04-05T03:12:52.262555+0000","last_fullsized":"2024-04-05T03:12:52.262555+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:50:50.549731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284640+0000","last_change":"2024-04-05T03:12:53.284640+0000","last_active":"2024-04-05T03:12:53.284640+0000","last_peered":"2024-04-05T03:12:53.284640+0000","last_clean":"2024-04-05T03:12:53.284640+0000","last_became_active":"2024-04-05T03:12:53.283863+0000","last_became_peered":"2024-04-05T03:12:53.283863+0000","last_unstale":"2024-04-05T03:12:53.284640+0000","last_undegraded":"2024-04-05T03:12:53.284640+0000","last_fullsized":"2024-04-05T03:12:53.284640+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:08:49.107166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"88'161","reported_seq":449,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266125+0000","last_change":"2024-04-05T03:10:31.862851+0000","last_active":"2024-04-05T03:12:52.266125+0000","last_peered":"2024-04-05T03:12:52.266125+0000","last_clean":"2024-04-05T03:12:52.266125+0000","last_became_active":"2024-04-05T03:10:31.861865+0000","last_became_peered":"2024-04-05T03:10:31.861865+0000","last_unstale":"2024-04-05T03:12:52.266125+0000","last_undegraded":"2024-04-05T03:12:52.266125+0000","last_fullsized":"2024-04-05T03:12:52.266125+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:05:19.760756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"84'199","reported_seq":717,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258768+0000","last_change":"2024-04-05T03:12:18.830679+0000","last_active":"2024-04-05T03:12:52.258768+0000","last_peered":"2024-04-05T03:12:52.258768+0000","last_clean":"2024-04-05T03:12:52.258768+0000","last_became_active":"2024-04-05T03:12:18.830053+0000","last_became_peered":"2024-04-05T03:12:18.830053+0000","last_unstale":"2024-04-05T03:12:52.258768+0000","last_undegraded":"2024-04-05T03:12:52.258768+0000","last_fullsized":"2024-04-05T03:12:52.258768+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:41:46.843315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041847,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1018,"num_read_kb":1289,"num_write":171,"num_write_kb":1317,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271845+0000","last_change":"2024-04-05T03:11:44.819843+0000","last_active":"2024-04-05T03:12:53.271845+0000","last_peered":"2024-04-05T03:12:53.271845+0000","last_clean":"2024-04-05T03:12:53.271845+0000","last_became_active":"2024-04-05T03:11:44.819365+0000","last_became_peered":"2024-04-05T03:11:44.819365+0000","last_unstale":"2024-04-05T03:12:53.271845+0000","last_undegraded":"2024-04-05T03:12:53.271845+0000","last_fullsized":"2024-04-05T03:12:53.271845+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:04:22.836425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.270431+0000","last_change":"2024-04-05T03:12:18.831877+0000","last_active":"2024-04-05T03:12:53.270431+0000","last_peered":"2024-04-05T03:12:53.270431+0000","last_clean":"2024-04-05T03:12:53.270431+0000","last_became_active":"2024-04-05T03:12:18.831529+0000","last_became_peered":"2024-04-05T03:12:18.831529+0000","last_unstale":"2024-04-05T03:12:53.270431+0000","last_undegraded":"2024-04-05T03:12:53.270431+0000","last_fullsized":"2024-04-05T03:12:53.270431+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:11:32.246265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"84'150","reported_seq":693,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.279302+0000","last_change":"2024-04-05T03:12:53.279302+0000","last_active":"2024-04-05T03:12:53.279302+0000","last_peered":"2024-04-05T03:12:53.279302+0000","last_clean":"2024-04-05T03:12:53.279302+0000","last_became_active":"2024-04-05T03:12:53.278637+0000","last_became_peered":"2024-04-05T03:12:53.278637+0000","last_unstale":"2024-04-05T03:12:53.279302+0000","last_undegraded":"2024-04-05T03:12:53.279302+0000","last_fullsized":"2024-04-05T03:12:53.279302+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:02:40.892179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032387699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1041,"num_read_kb":3170,"num_write":130,"num_write_kb":2261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288343+0000","last_change":"2024-04-05T03:12:53.288056+0000","last_active":"2024-04-05T03:12:53.288343+0000","last_peered":"2024-04-05T03:12:53.288343+0000","last_clean":"2024-04-05T03:12:53.288343+0000","last_became_active":"2024-04-05T03:12:53.287593+0000","last_became_peered":"2024-04-05T03:12:53.287593+0000","last_unstale":"2024-04-05T03:12:53.288343+0000","last_undegraded":"2024-04-05T03:12:53.288343+0000","last_fullsized":"2024-04-05T03:12:53.288343+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:37:51.955764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265989+0000","last_change":"2024-04-05T03:10:31.868757+0000","last_active":"2024-04-05T03:12:52.265989+0000","last_peered":"2024-04-05T03:12:52.265989+0000","last_clean":"2024-04-05T03:12:52.265989+0000","last_became_active":"2024-04-05T03:10:31.868505+0000","last_became_peered":"2024-04-05T03:10:31.868505+0000","last_unstale":"2024-04-05T03:12:52.265989+0000","last_undegraded":"2024-04-05T03:12:52.265989+0000","last_fullsized":"2024-04-05T03:12:52.265989+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:32:37.077890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'125","reported_seq":793,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258739+0000","last_change":"2024-04-05T03:11:44.823007+0000","last_active":"2024-04-05T03:12:52.258739+0000","last_peered":"2024-04-05T03:12:52.258739+0000","last_clean":"2024-04-05T03:12:52.258739+0000","last_became_active":"2024-04-05T03:11:44.822476+0000","last_became_peered":"2024-04-05T03:11:44.822476+0000","last_unstale":"2024-04-05T03:12:52.258739+0000","last_undegraded":"2024-04-05T03:12:52.258739+0000","last_fullsized":"2024-04-05T03:12:52.258739+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:23:21.702987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053167100000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1152,"num_read_kb":1185,"num_write":104,"num_write_kb":138,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.270938+0000","last_change":"2024-04-05T03:11:44.824231+0000","last_active":"2024-04-05T03:12:53.270938+0000","last_peered":"2024-04-05T03:12:53.270938+0000","last_clean":"2024-04-05T03:12:53.270938+0000","last_became_active":"2024-04-05T03:11:44.823540+0000","last_became_peered":"2024-04-05T03:11:44.823540+0000","last_unstale":"2024-04-05T03:12:53.270938+0000","last_undegraded":"2024-04-05T03:12:53.270938+0000","last_fullsized":"2024-04-05T03:12:53.270938+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:47:47.664620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281466+0000","last_change":"2024-04-05T03:12:53.281466+0000","last_active":"2024-04-05T03:12:53.281466+0000","last_peered":"2024-04-05T03:12:53.281466+0000","last_clean":"2024-04-05T03:12:53.281466+0000","last_became_active":"2024-04-05T03:12:53.280200+0000","last_became_peered":"2024-04-05T03:12:53.280200+0000","last_unstale":"2024-04-05T03:12:53.281466+0000","last_undegraded":"2024-04-05T03:12:53.281466+0000","last_fullsized":"2024-04-05T03:12:53.281466+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:10:15.421941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"84'133","reported_seq":532,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271381+0000","last_change":"2024-04-05T03:11:44.822006+0000","last_active":"2024-04-05T03:12:53.271381+0000","last_peered":"2024-04-05T03:12:53.271381+0000","last_clean":"2024-04-05T03:12:53.271381+0000","last_became_active":"2024-04-05T03:11:44.821420+0000","last_became_peered":"2024-04-05T03:11:44.821420+0000","last_unstale":"2024-04-05T03:12:53.271381+0000","last_undegraded":"2024-04-05T03:12:53.271381+0000","last_fullsized":"2024-04-05T03:12:53.271381+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":133,"log_dups_size":0,"ondisk_log_size":133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:40:41.011945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032491300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":648,"num_read_kb":1690,"num_write":135,"num_write_kb":1133,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284068+0000","last_change":"2024-04-05T03:12:53.284068+0000","last_active":"2024-04-05T03:12:53.284068+0000","last_peered":"2024-04-05T03:12:53.284068+0000","last_clean":"2024-04-05T03:12:53.284068+0000","last_became_active":"2024-04-05T03:12:53.283456+0000","last_became_peered":"2024-04-05T03:12:53.283456+0000","last_unstale":"2024-04-05T03:12:53.284068+0000","last_undegraded":"2024-04-05T03:12:53.284068+0000","last_fullsized":"2024-04-05T03:12:53.284068+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:31:16.994153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288318+0000","last_change":"2024-04-05T03:12:53.288318+0000","last_active":"2024-04-05T03:12:53.288318+0000","last_peered":"2024-04-05T03:12:53.288318+0000","last_clean":"2024-04-05T03:12:53.288318+0000","last_became_active":"2024-04-05T03:12:53.287961+0000","last_became_peered":"2024-04-05T03:12:53.287961+0000","last_unstale":"2024-04-05T03:12:53.288318+0000","last_undegraded":"2024-04-05T03:12:53.288318+0000","last_fullsized":"2024-04-05T03:12:53.288318+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:16:26.389764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266224+0000","last_change":"2024-04-05T03:11:44.824152+0000","last_active":"2024-04-05T03:12:52.266224+0000","last_peered":"2024-04-05T03:12:52.266224+0000","last_clean":"2024-04-05T03:12:52.266224+0000","last_became_active":"2024-04-05T03:11:44.823842+0000","last_became_peered":"2024-04-05T03:11:44.823842+0000","last_unstale":"2024-04-05T03:12:52.266224+0000","last_undegraded":"2024-04-05T03:12:52.266224+0000","last_fullsized":"2024-04-05T03:12:52.266224+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:26:31.940086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267009+0000","last_change":"2024-04-05T03:12:18.834468+0000","last_active":"2024-04-05T03:12:52.267009+0000","last_peered":"2024-04-05T03:12:52.267009+0000","last_clean":"2024-04-05T03:12:52.267009+0000","last_became_active":"2024-04-05T03:12:18.834140+0000","last_became_peered":"2024-04-05T03:12:18.834140+0000","last_unstale":"2024-04-05T03:12:52.267009+0000","last_undegraded":"2024-04-05T03:12:52.267009+0000","last_fullsized":"2024-04-05T03:12:52.267009+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:37:04.934029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266686+0000","last_change":"2024-04-05T03:12:18.834695+0000","last_active":"2024-04-05T03:12:52.266686+0000","last_peered":"2024-04-05T03:12:52.266686+0000","last_clean":"2024-04-05T03:12:52.266686+0000","last_became_active":"2024-04-05T03:12:18.834415+0000","last_became_peered":"2024-04-05T03:12:18.834415+0000","last_unstale":"2024-04-05T03:12:52.266686+0000","last_undegraded":"2024-04-05T03:12:52.266686+0000","last_fullsized":"2024-04-05T03:12:52.266686+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:07:47.977921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.263798+0000","last_change":"2024-04-05T03:11:44.829880+0000","last_active":"2024-04-05T03:12:52.263798+0000","last_peered":"2024-04-05T03:12:52.263798+0000","last_clean":"2024-04-05T03:12:52.263798+0000","last_became_active":"2024-04-05T03:11:44.829552+0000","last_became_peered":"2024-04-05T03:11:44.829552+0000","last_unstale":"2024-04-05T03:12:52.263798+0000","last_undegraded":"2024-04-05T03:12:52.263798+0000","last_fullsized":"2024-04-05T03:12:52.263798+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:36:37.307694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283661+0000","last_change":"2024-04-05T03:12:53.283661+0000","last_active":"2024-04-05T03:12:53.283661+0000","last_peered":"2024-04-05T03:12:53.283661+0000","last_clean":"2024-04-05T03:12:53.283661+0000","last_became_active":"2024-04-05T03:12:53.283271+0000","last_became_peered":"2024-04-05T03:12:53.283271+0000","last_unstale":"2024-04-05T03:12:53.283661+0000","last_undegraded":"2024-04-05T03:12:53.283661+0000","last_fullsized":"2024-04-05T03:12:53.283661+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:38:10.656186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288690+0000","last_change":"2024-04-05T03:12:53.288690+0000","last_active":"2024-04-05T03:12:53.288690+0000","last_peered":"2024-04-05T03:12:53.288690+0000","last_clean":"2024-04-05T03:12:53.288690+0000","last_became_active":"2024-04-05T03:12:53.287105+0000","last_became_peered":"2024-04-05T03:12:53.287105+0000","last_unstale":"2024-04-05T03:12:53.288690+0000","last_undegraded":"2024-04-05T03:12:53.288690+0000","last_fullsized":"2024-04-05T03:12:53.288690+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:45:04.692793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271934+0000","last_change":"2024-04-05T03:11:44.822882+0000","last_active":"2024-04-05T03:12:53.271934+0000","last_peered":"2024-04-05T03:12:53.271934+0000","last_clean":"2024-04-05T03:12:53.271934+0000","last_became_active":"2024-04-05T03:11:44.822543+0000","last_became_peered":"2024-04-05T03:11:44.822543+0000","last_unstale":"2024-04-05T03:12:53.271934+0000","last_undegraded":"2024-04-05T03:12:53.271934+0000","last_fullsized":"2024-04-05T03:12:53.271934+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:22:12.438776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284191+0000","last_change":"2024-04-05T03:12:53.284191+0000","last_active":"2024-04-05T03:12:53.284191+0000","last_peered":"2024-04-05T03:12:53.284191+0000","last_clean":"2024-04-05T03:12:53.284191+0000","last_became_active":"2024-04-05T03:12:53.283859+0000","last_became_peered":"2024-04-05T03:12:53.283859+0000","last_unstale":"2024-04-05T03:12:53.284191+0000","last_undegraded":"2024-04-05T03:12:53.284191+0000","last_fullsized":"2024-04-05T03:12:53.284191+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:49:28.905998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.263640+0000","last_change":"2024-04-05T03:12:18.831147+0000","last_active":"2024-04-05T03:12:52.263640+0000","last_peered":"2024-04-05T03:12:52.263640+0000","last_clean":"2024-04-05T03:12:52.263640+0000","last_became_active":"2024-04-05T03:12:18.830779+0000","last_became_peered":"2024-04-05T03:12:18.830779+0000","last_unstale":"2024-04-05T03:12:52.263640+0000","last_undegraded":"2024-04-05T03:12:52.263640+0000","last_fullsized":"2024-04-05T03:12:52.263640+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:18:18.393332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":53,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.264337+0000","last_change":"2024-04-05T03:02:48.461252+0000","last_active":"2024-04-05T03:12:52.264337+0000","last_peered":"2024-04-05T03:12:52.264337+0000","last_clean":"2024-04-05T03:12:52.264337+0000","last_became_active":"2024-04-05T03:02:48.460982+0000","last_became_peered":"2024-04-05T03:02:48.460982+0000","last_unstale":"2024-04-05T03:12:52.264337+0000","last_undegraded":"2024-04-05T03:12:52.264337+0000","last_fullsized":"2024-04-05T03:12:52.264337+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:03.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286911+0000","last_change":"2024-04-05T03:12:53.286911+0000","last_active":"2024-04-05T03:12:53.286911+0000","last_peered":"2024-04-05T03:12:53.286911+0000","last_clean":"2024-04-05T03:12:53.286911+0000","last_became_active":"2024-04-05T03:12:53.286680+0000","last_became_peered":"2024-04-05T03:12:53.286680+0000","last_unstale":"2024-04-05T03:12:53.286911+0000","last_undegraded":"2024-04-05T03:12:53.286911+0000","last_fullsized":"2024-04-05T03:12:53.286911+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:04:27.036557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":71,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.264346+0000","last_change":"2024-04-05T03:11:44.822325+0000","last_active":"2024-04-05T03:12:52.264346+0000","last_peered":"2024-04-05T03:12:52.264346+0000","last_clean":"2024-04-05T03:12:52.264346+0000","last_became_active":"2024-04-05T03:11:44.821851+0000","last_became_peered":"2024-04-05T03:11:44.821851+0000","last_unstale":"2024-04-05T03:12:52.264346+0000","last_undegraded":"2024-04-05T03:12:52.264346+0000","last_fullsized":"2024-04-05T03:12:52.264346+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:34:03.316393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258716+0000","last_change":"2024-04-05T03:02:48.903134+0000","last_active":"2024-04-05T03:12:52.258716+0000","last_peered":"2024-04-05T03:12:52.258716+0000","last_clean":"2024-04-05T03:12:52.258716+0000","last_became_active":"2024-04-05T03:02:48.902836+0000","last_became_peered":"2024-04-05T03:02:48.902836+0000","last_unstale":"2024-04-05T03:12:52.258716+0000","last_undegraded":"2024-04-05T03:12:52.258716+0000","last_fullsized":"2024-04-05T03:12:52.258716+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:08:54.250101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":70,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265091+0000","last_change":"2024-04-05T03:10:31.862564+0000","last_active":"2024-04-05T03:12:52.265091+0000","last_peered":"2024-04-05T03:12:52.265091+0000","last_clean":"2024-04-05T03:12:52.265091+0000","last_became_active":"2024-04-05T03:10:31.862202+0000","last_became_peered":"2024-04-05T03:10:31.862202+0000","last_unstale":"2024-04-05T03:12:52.265091+0000","last_undegraded":"2024-04-05T03:12:52.265091+0000","last_fullsized":"2024-04-05T03:12:52.265091+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:34:22.600326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.289926+0000","last_change":"2024-04-05T03:12:53.289926+0000","last_active":"2024-04-05T03:12:53.289926+0000","last_peered":"2024-04-05T03:12:53.289926+0000","last_clean":"2024-04-05T03:12:53.289926+0000","last_became_active":"2024-04-05T03:12:53.288900+0000","last_became_peered":"2024-04-05T03:12:53.288900+0000","last_unstale":"2024-04-05T03:12:53.289926+0000","last_undegraded":"2024-04-05T03:12:53.289926+0000","last_fullsized":"2024-04-05T03:12:53.289926+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:21:50.597917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":77,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.261525+0000","last_change":"2024-04-05T03:10:31.860733+0000","last_active":"2024-04-05T03:12:52.261525+0000","last_peered":"2024-04-05T03:12:52.261525+0000","last_clean":"2024-04-05T03:12:52.261525+0000","last_became_active":"2024-04-05T03:10:31.860273+0000","last_became_peered":"2024-04-05T03:10:31.860273+0000","last_unstale":"2024-04-05T03:12:52.261525+0000","last_undegraded":"2024-04-05T03:12:52.261525+0000","last_fullsized":"2024-04-05T03:12:52.261525+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:05:04.356023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283189+0000","last_change":"2024-04-05T03:12:53.282806+0000","last_active":"2024-04-05T03:12:53.283189+0000","last_peered":"2024-04-05T03:12:53.283189+0000","last_clean":"2024-04-05T03:12:53.283189+0000","last_became_active":"2024-04-05T03:12:53.282476+0000","last_became_peered":"2024-04-05T03:12:53.282476+0000","last_unstale":"2024-04-05T03:12:53.283189+0000","last_undegraded":"2024-04-05T03:12:53.283189+0000","last_fullsized":"2024-04-05T03:12:53.283189+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:20:50.423294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.263585+0000","last_change":"2024-04-05T03:02:48.460853+0000","last_active":"2024-04-05T03:12:52.263585+0000","last_peered":"2024-04-05T03:12:52.263585+0000","last_clean":"2024-04-05T03:12:52.263585+0000","last_became_active":"2024-04-05T03:02:48.460557+0000","last_became_peered":"2024-04-05T03:02:48.460557+0000","last_unstale":"2024-04-05T03:12:52.263585+0000","last_undegraded":"2024-04-05T03:12:52.263585+0000","last_fullsized":"2024-04-05T03:12:52.263585+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:57.261993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281677+0000","last_change":"2024-04-05T03:12:53.281677+0000","last_active":"2024-04-05T03:12:53.281677+0000","last_peered":"2024-04-05T03:12:53.281677+0000","last_clean":"2024-04-05T03:12:53.281677+0000","last_became_active":"2024-04-05T03:12:53.280903+0000","last_became_peered":"2024-04-05T03:12:53.280903+0000","last_unstale":"2024-04-05T03:12:53.281677+0000","last_undegraded":"2024-04-05T03:12:53.281677+0000","last_fullsized":"2024-04-05T03:12:53.281677+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:56:09.930244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281646+0000","last_change":"2024-04-05T03:12:53.281646+0000","last_active":"2024-04-05T03:12:53.281646+0000","last_peered":"2024-04-05T03:12:53.281646+0000","last_clean":"2024-04-05T03:12:53.281646+0000","last_became_active":"2024-04-05T03:12:53.281333+0000","last_became_peered":"2024-04-05T03:12:53.281333+0000","last_unstale":"2024-04-05T03:12:53.281646+0000","last_undegraded":"2024-04-05T03:12:53.281646+0000","last_fullsized":"2024-04-05T03:12:53.281646+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:20:08.398538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.289627+0000","last_change":"2024-04-05T03:12:53.289627+0000","last_active":"2024-04-05T03:12:53.289627+0000","last_peered":"2024-04-05T03:12:53.289627+0000","last_clean":"2024-04-05T03:12:53.289627+0000","last_became_active":"2024-04-05T03:12:53.289111+0000","last_became_peered":"2024-04-05T03:12:53.289111+0000","last_unstale":"2024-04-05T03:12:53.289627+0000","last_undegraded":"2024-04-05T03:12:53.289627+0000","last_fullsized":"2024-04-05T03:12:53.289627+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:56:31.406641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":71,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.262796+0000","last_change":"2024-04-05T03:12:18.833440+0000","last_active":"2024-04-05T03:12:52.262796+0000","last_peered":"2024-04-05T03:12:52.262796+0000","last_clean":"2024-04-05T03:12:52.262796+0000","last_became_active":"2024-04-05T03:12:18.833168+0000","last_became_peered":"2024-04-05T03:12:18.833168+0000","last_unstale":"2024-04-05T03:12:52.262796+0000","last_undegraded":"2024-04-05T03:12:52.262796+0000","last_fullsized":"2024-04-05T03:12:52.262796+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:36.928082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266296+0000","last_change":"2024-04-05T03:11:44.827920+0000","last_active":"2024-04-05T03:12:52.266296+0000","last_peered":"2024-04-05T03:12:52.266296+0000","last_clean":"2024-04-05T03:12:52.266296+0000","last_became_active":"2024-04-05T03:11:44.827537+0000","last_became_peered":"2024-04-05T03:11:44.827537+0000","last_unstale":"2024-04-05T03:12:52.266296+0000","last_undegraded":"2024-04-05T03:12:52.266296+0000","last_fullsized":"2024-04-05T03:12:52.266296+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:59:25.074973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"88'22","reported_seq":74,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267978+0000","last_change":"2024-04-05T03:02:48.455833+0000","last_active":"2024-04-05T03:12:52.267978+0000","last_peered":"2024-04-05T03:12:52.267978+0000","last_clean":"2024-04-05T03:12:52.267978+0000","last_became_active":"2024-04-05T03:02:48.455185+0000","last_became_peered":"2024-04-05T03:02:48.455185+0000","last_unstale":"2024-04-05T03:12:52.267978+0000","last_undegraded":"2024-04-05T03:12:52.267978+0000","last_fullsized":"2024-04-05T03:12:52.267978+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:02.621579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.270987+0000","last_change":"2024-04-05T03:12:18.831110+0000","last_active":"2024-04-05T03:12:53.270987+0000","last_peered":"2024-04-05T03:12:53.270987+0000","last_clean":"2024-04-05T03:12:53.270987+0000","last_became_active":"2024-04-05T03:12:18.830763+0000","last_became_peered":"2024-04-05T03:12:18.830763+0000","last_unstale":"2024-04-05T03:12:53.270987+0000","last_undegraded":"2024-04-05T03:12:53.270987+0000","last_fullsized":"2024-04-05T03:12:53.270987+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:07:13.205635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288134+0000","last_change":"2024-04-05T03:12:53.288134+0000","last_active":"2024-04-05T03:12:53.288134+0000","last_peered":"2024-04-05T03:12:53.288134+0000","last_clean":"2024-04-05T03:12:53.288134+0000","last_became_active":"2024-04-05T03:12:53.287780+0000","last_became_peered":"2024-04-05T03:12:53.287780+0000","last_unstale":"2024-04-05T03:12:53.288134+0000","last_undegraded":"2024-04-05T03:12:53.288134+0000","last_fullsized":"2024-04-05T03:12:53.288134+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:48:29.253792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286200+0000","last_change":"2024-04-05T03:12:53.286200+0000","last_active":"2024-04-05T03:12:53.286200+0000","last_peered":"2024-04-05T03:12:53.286200+0000","last_clean":"2024-04-05T03:12:53.286200+0000","last_became_active":"2024-04-05T03:12:53.285836+0000","last_became_peered":"2024-04-05T03:12:53.285836+0000","last_unstale":"2024-04-05T03:12:53.286200+0000","last_undegraded":"2024-04-05T03:12:53.286200+0000","last_fullsized":"2024-04-05T03:12:53.286200+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:32:39.977616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258955+0000","last_change":"2024-04-05T03:11:44.821345+0000","last_active":"2024-04-05T03:12:52.258955+0000","last_peered":"2024-04-05T03:12:52.258955+0000","last_clean":"2024-04-05T03:12:52.258955+0000","last_became_active":"2024-04-05T03:11:44.821003+0000","last_became_peered":"2024-04-05T03:11:44.821003+0000","last_unstale":"2024-04-05T03:12:52.258955+0000","last_undegraded":"2024-04-05T03:12:52.258955+0000","last_fullsized":"2024-04-05T03:12:52.258955+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:08:49.149318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.289139+0000","last_change":"2024-04-05T03:12:53.289139+0000","last_active":"2024-04-05T03:12:53.289139+0000","last_peered":"2024-04-05T03:12:53.289139+0000","last_clean":"2024-04-05T03:12:53.289139+0000","last_became_active":"2024-04-05T03:12:53.288830+0000","last_became_peered":"2024-04-05T03:12:53.288830+0000","last_unstale":"2024-04-05T03:12:53.289139+0000","last_undegraded":"2024-04-05T03:12:53.289139+0000","last_fullsized":"2024-04-05T03:12:53.289139+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:15:45.481068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281901+0000","last_change":"2024-04-05T03:12:53.281302+0000","last_active":"2024-04-05T03:12:53.281901+0000","last_peered":"2024-04-05T03:12:53.281901+0000","last_clean":"2024-04-05T03:12:53.281901+0000","last_became_active":"2024-04-05T03:12:53.280958+0000","last_became_peered":"2024-04-05T03:12:53.280958+0000","last_unstale":"2024-04-05T03:12:53.281901+0000","last_undegraded":"2024-04-05T03:12:53.281901+0000","last_fullsized":"2024-04-05T03:12:53.281901+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:35:26.995968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267501+0000","last_change":"2024-04-05T03:12:18.835520+0000","last_active":"2024-04-05T03:12:52.267501+0000","last_peered":"2024-04-05T03:12:52.267501+0000","last_clean":"2024-04-05T03:12:52.267501+0000","last_became_active":"2024-04-05T03:12:18.834156+0000","last_became_peered":"2024-04-05T03:12:18.834156+0000","last_unstale":"2024-04-05T03:12:52.267501+0000","last_undegraded":"2024-04-05T03:12:52.267501+0000","last_fullsized":"2024-04-05T03:12:52.267501+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:36:18.106186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288550+0000","last_change":"2024-04-05T03:12:53.288550+0000","last_active":"2024-04-05T03:12:53.288550+0000","last_peered":"2024-04-05T03:12:53.288550+0000","last_clean":"2024-04-05T03:12:53.288550+0000","last_became_active":"2024-04-05T03:12:53.288240+0000","last_became_peered":"2024-04-05T03:12:53.288240+0000","last_unstale":"2024-04-05T03:12:53.288550+0000","last_undegraded":"2024-04-05T03:12:53.288550+0000","last_fullsized":"2024-04-05T03:12:53.288550+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:09:23.144915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284139+0000","last_change":"2024-04-05T03:12:53.284139+0000","last_active":"2024-04-05T03:12:53.284139+0000","last_peered":"2024-04-05T03:12:53.284139+0000","last_clean":"2024-04-05T03:12:53.284139+0000","last_became_active":"2024-04-05T03:12:53.283489+0000","last_became_peered":"2024-04-05T03:12:53.283489+0000","last_unstale":"2024-04-05T03:12:53.284139+0000","last_undegraded":"2024-04-05T03:12:53.284139+0000","last_fullsized":"2024-04-05T03:12:53.284139+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:55:01.202431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283162+0000","last_change":"2024-04-05T03:12:53.283162+0000","last_active":"2024-04-05T03:12:53.283162+0000","last_peered":"2024-04-05T03:12:53.283162+0000","last_clean":"2024-04-05T03:12:53.283162+0000","last_became_active":"2024-04-05T03:12:53.282796+0000","last_became_peered":"2024-04-05T03:12:53.282796+0000","last_unstale":"2024-04-05T03:12:53.283162+0000","last_undegraded":"2024-04-05T03:12:53.283162+0000","last_fullsized":"2024-04-05T03:12:53.283162+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:44:35.054496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":36,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.282737+0000","last_change":"2024-04-05T03:12:53.282737+0000","last_active":"2024-04-05T03:12:53.282737+0000","last_peered":"2024-04-05T03:12:53.282737+0000","last_clean":"2024-04-05T03:12:53.282737+0000","last_became_active":"2024-04-05T03:12:53.281638+0000","last_became_peered":"2024-04-05T03:12:53.281638+0000","last_unstale":"2024-04-05T03:12:53.282737+0000","last_undegraded":"2024-04-05T03:12:53.282737+0000","last_fullsized":"2024-04-05T03:12:53.282737+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:37.735394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":87,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265893+0000","last_change":"2024-04-05T03:12:18.834654+0000","last_active":"2024-04-05T03:12:52.265893+0000","last_peered":"2024-04-05T03:12:52.265893+0000","last_clean":"2024-04-05T03:12:52.265893+0000","last_became_active":"2024-04-05T03:12:18.834305+0000","last_became_peered":"2024-04-05T03:12:18.834305+0000","last_unstale":"2024-04-05T03:12:52.265893+0000","last_undegraded":"2024-04-05T03:12:52.265893+0000","last_fullsized":"2024-04-05T03:12:52.265893+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:13:51.558689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18023,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":12992,"num_write":846,"num_write_kb":12586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5428,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201435,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307224,"kb_used_data":5772,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93437928,"statfs":{"total":95995035648,"available":95680438272,"internally_reserved":0,"allocated":5910528,"data_stored":5387602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463072,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303120,"kb_used_data":1664,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93442032,"statfs":{"total":95995035648,"available":95684640768,"internally_reserved":0,"allocated":1703936,"data_stored":1192927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":97,"seq":416611827739,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32824,"kb_used_data":1124,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93712328,"statfs":{"total":95995035648,"available":95961423872,"internally_reserved":0,"allocated":1150976,"data_stored":608317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8054,"internal_metadata":32432266},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":92,"seq":395136991274,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37508,"kb_used_data":5816,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93707644,"statfs":{"total":95995035648,"available":95956627456,"internally_reserved":0,"allocated":5955584,"data_stored":5394175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7741,"internal_metadata":32432579},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":102,"seq":438086664212,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1112,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1138688,"data_stored":605169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8171,"internal_metadata":28172309},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":107,"seq":459561500685,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28632,"kb_used_data":1092,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716520,"statfs":{"total":95995035648,"available":95965716480,"internally_reserved":0,"allocated":1118208,"data_stored":602088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6301,"internal_metadata":28174179},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986345,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302544,"kb_used_data":5188,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442608,"statfs":{"total":95995035648,"available":95685230592,"internally_reserved":0,"allocated":5312512,"data_stored":4797164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724708,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298440,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446712,"statfs":{"total":95995035648,"available":95689433088,"internally_reserved":0,"allocated":1105920,"data_stored":598629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3845,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1583,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6037,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6350,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2622,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3014,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:13:52.076 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:13:52.343 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:52.444 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:52 smithi033 ceph-mon[97738]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:52.444 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:52 smithi033 ceph-mon[97738]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:13:52.445 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:52 smithi033 ceph-mon[101783]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:52.445 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:52 smithi033 ceph-mon[101783]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:13:52.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:52 smithi151 ceph-mon[88672]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:52.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:52 smithi151 ceph-mon[88672]: from='client.44522 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:13:54.154 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:13:54.155 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-04-05T03:13:54.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:54 smithi033 ceph-mon[97738]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:54.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:54 smithi033 ceph-mon[101783]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:54.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:54 smithi151 ceph-mon[88672]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:54.863 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":242,"stamp":"2024-04-05T03:13:53.106495+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":13916,"num_write":1146,"num_write_kb":15031,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1338940,"kb_used_data":22848,"kb_used_omap":36,"kb_used_meta":1315931,"kb_avail":748622276,"statfs":{"total":767960285184,"available":766589210624,"internally_reserved":0,"allocated":23396352,"data_stored":19186071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37083,"internal_metadata":1347514149},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002627"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265583+0000","last_change":"2024-04-05T03:11:44.826962+0000","last_active":"2024-04-05T03:12:52.265583+0000","last_peered":"2024-04-05T03:12:52.265583+0000","last_clean":"2024-04-05T03:12:52.265583+0000","last_became_active":"2024-04-05T03:11:44.826572+0000","last_became_peered":"2024-04-05T03:11:44.826572+0000","last_unstale":"2024-04-05T03:12:52.265583+0000","last_undegraded":"2024-04-05T03:12:52.265583+0000","last_fullsized":"2024-04-05T03:12:52.265583+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:28:53.091252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272871+0000","last_change":"2024-04-05T03:12:18.831902+0000","last_active":"2024-04-05T03:12:53.272871+0000","last_peered":"2024-04-05T03:12:53.272871+0000","last_clean":"2024-04-05T03:12:53.272871+0000","last_became_active":"2024-04-05T03:12:18.831510+0000","last_became_peered":"2024-04-05T03:12:18.831510+0000","last_unstale":"2024-04-05T03:12:53.272871+0000","last_undegraded":"2024-04-05T03:12:53.272871+0000","last_fullsized":"2024-04-05T03:12:53.272871+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:41:57.365084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.287316+0000","last_change":"2024-04-05T03:12:53.287316+0000","last_active":"2024-04-05T03:12:53.287316+0000","last_peered":"2024-04-05T03:12:53.287316+0000","last_clean":"2024-04-05T03:12:53.287316+0000","last_became_active":"2024-04-05T03:12:53.286740+0000","last_became_peered":"2024-04-05T03:12:53.286740+0000","last_unstale":"2024-04-05T03:12:53.287316+0000","last_undegraded":"2024-04-05T03:12:53.287316+0000","last_fullsized":"2024-04-05T03:12:53.287316+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:31:51.207997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":77,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.285988+0000","last_change":"2024-04-05T03:12:53.285988+0000","last_active":"2024-04-05T03:12:53.285988+0000","last_peered":"2024-04-05T03:12:53.285988+0000","last_clean":"2024-04-05T03:12:53.285988+0000","last_became_active":"2024-04-05T03:12:53.285634+0000","last_became_peered":"2024-04-05T03:12:53.285634+0000","last_unstale":"2024-04-05T03:12:53.285988+0000","last_undegraded":"2024-04-05T03:12:53.285988+0000","last_fullsized":"2024-04-05T03:12:53.285988+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:00:45.210744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283853+0000","last_change":"2024-04-05T03:12:53.283853+0000","last_active":"2024-04-05T03:12:53.283853+0000","last_peered":"2024-04-05T03:12:53.283853+0000","last_clean":"2024-04-05T03:12:53.283853+0000","last_became_active":"2024-04-05T03:12:53.283470+0000","last_became_peered":"2024-04-05T03:12:53.283470+0000","last_unstale":"2024-04-05T03:12:53.283853+0000","last_undegraded":"2024-04-05T03:12:53.283853+0000","last_fullsized":"2024-04-05T03:12:53.283853+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:07:07.647098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":54,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.260283+0000","last_change":"2024-04-05T03:02:48.458396+0000","last_active":"2024-04-05T03:12:52.260283+0000","last_peered":"2024-04-05T03:12:52.260283+0000","last_clean":"2024-04-05T03:12:52.260283+0000","last_became_active":"2024-04-05T03:02:48.458092+0000","last_became_peered":"2024-04-05T03:02:48.458092+0000","last_unstale":"2024-04-05T03:12:52.260283+0000","last_undegraded":"2024-04-05T03:12:52.260283+0000","last_fullsized":"2024-04-05T03:12:52.260283+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:43.973088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283628+0000","last_change":"2024-04-05T03:12:53.283628+0000","last_active":"2024-04-05T03:12:53.283628+0000","last_peered":"2024-04-05T03:12:53.283628+0000","last_clean":"2024-04-05T03:12:53.283628+0000","last_became_active":"2024-04-05T03:12:53.283299+0000","last_became_peered":"2024-04-05T03:12:53.283299+0000","last_unstale":"2024-04-05T03:12:53.283628+0000","last_undegraded":"2024-04-05T03:12:53.283628+0000","last_fullsized":"2024-04-05T03:12:53.283628+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:16:07.905585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":71,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258823+0000","last_change":"2024-04-05T03:11:44.823272+0000","last_active":"2024-04-05T03:12:52.258823+0000","last_peered":"2024-04-05T03:12:52.258823+0000","last_clean":"2024-04-05T03:12:52.258823+0000","last_became_active":"2024-04-05T03:11:44.822883+0000","last_became_peered":"2024-04-05T03:11:44.822883+0000","last_unstale":"2024-04-05T03:12:52.258823+0000","last_undegraded":"2024-04-05T03:12:52.258823+0000","last_fullsized":"2024-04-05T03:12:52.258823+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:22:55.498909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283492+0000","last_change":"2024-04-05T03:12:53.283492+0000","last_active":"2024-04-05T03:12:53.283492+0000","last_peered":"2024-04-05T03:12:53.283492+0000","last_clean":"2024-04-05T03:12:53.283492+0000","last_became_active":"2024-04-05T03:12:53.283178+0000","last_became_peered":"2024-04-05T03:12:53.283178+0000","last_unstale":"2024-04-05T03:12:53.283492+0000","last_undegraded":"2024-04-05T03:12:53.283492+0000","last_fullsized":"2024-04-05T03:12:53.283492+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:36:55.383542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":57,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272082+0000","last_change":"2024-04-05T03:10:31.867161+0000","last_active":"2024-04-05T03:12:53.272082+0000","last_peered":"2024-04-05T03:12:53.272082+0000","last_clean":"2024-04-05T03:12:53.272082+0000","last_became_active":"2024-04-05T03:10:31.866213+0000","last_became_peered":"2024-04-05T03:10:31.866213+0000","last_unstale":"2024-04-05T03:12:53.272082+0000","last_undegraded":"2024-04-05T03:12:53.272082+0000","last_fullsized":"2024-04-05T03:12:53.272082+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:09:11.571255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266774+0000","last_change":"2024-04-05T03:02:48.902933+0000","last_active":"2024-04-05T03:12:52.266774+0000","last_peered":"2024-04-05T03:12:52.266774+0000","last_clean":"2024-04-05T03:12:52.266774+0000","last_became_active":"2024-04-05T03:02:48.902583+0000","last_became_peered":"2024-04-05T03:02:48.902583+0000","last_unstale":"2024-04-05T03:12:52.266774+0000","last_undegraded":"2024-04-05T03:12:52.266774+0000","last_fullsized":"2024-04-05T03:12:52.266774+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:11:03.431547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272740+0000","last_change":"2024-04-05T03:11:44.824952+0000","last_active":"2024-04-05T03:12:53.272740+0000","last_peered":"2024-04-05T03:12:53.272740+0000","last_clean":"2024-04-05T03:12:53.272740+0000","last_became_active":"2024-04-05T03:11:44.824187+0000","last_became_peered":"2024-04-05T03:11:44.824187+0000","last_unstale":"2024-04-05T03:12:53.272740+0000","last_undegraded":"2024-04-05T03:12:53.272740+0000","last_fullsized":"2024-04-05T03:12:53.272740+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:39:01.880319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'50","reported_seq":175,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267530+0000","last_change":"2024-04-05T03:12:18.835306+0000","last_active":"2024-04-05T03:12:52.267530+0000","last_peered":"2024-04-05T03:12:52.267530+0000","last_clean":"2024-04-05T03:12:52.267530+0000","last_became_active":"2024-04-05T03:12:18.834232+0000","last_became_peered":"2024-04-05T03:12:18.834232+0000","last_unstale":"2024-04-05T03:12:52.267530+0000","last_undegraded":"2024-04-05T03:12:52.267530+0000","last_fullsized":"2024-04-05T03:12:52.267530+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":50,"log_dups_size":0,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:40:27.851343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036085400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":101,"num_read_kb":1566,"num_write":53,"num_write_kb":1295,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286266+0000","last_change":"2024-04-05T03:12:53.286266+0000","last_active":"2024-04-05T03:12:53.286266+0000","last_peered":"2024-04-05T03:12:53.286266+0000","last_clean":"2024-04-05T03:12:53.286266+0000","last_became_active":"2024-04-05T03:12:53.285928+0000","last_became_peered":"2024-04-05T03:12:53.285928+0000","last_unstale":"2024-04-05T03:12:53.286266+0000","last_undegraded":"2024-04-05T03:12:53.286266+0000","last_fullsized":"2024-04-05T03:12:53.286266+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:46:04.658838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'209","reported_seq":795,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272156+0000","last_change":"2024-04-05T03:11:44.824531+0000","last_active":"2024-04-05T03:12:53.272156+0000","last_peered":"2024-04-05T03:12:53.272156+0000","last_clean":"2024-04-05T03:12:53.272156+0000","last_became_active":"2024-04-05T03:11:44.823951+0000","last_became_peered":"2024-04-05T03:11:44.823951+0000","last_unstale":"2024-04-05T03:12:53.272156+0000","last_undegraded":"2024-04-05T03:12:53.272156+0000","last_fullsized":"2024-04-05T03:12:53.272156+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":209,"log_dups_size":0,"ondisk_log_size":209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:51.684899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074036299999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1296,"num_read_kb":1383,"num_write":166,"num_write_kb":238,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286319+0000","last_change":"2024-04-05T03:12:53.286319+0000","last_active":"2024-04-05T03:12:53.286319+0000","last_peered":"2024-04-05T03:12:53.286319+0000","last_clean":"2024-04-05T03:12:53.286319+0000","last_became_active":"2024-04-05T03:12:53.285932+0000","last_became_peered":"2024-04-05T03:12:53.285932+0000","last_unstale":"2024-04-05T03:12:53.286319+0000","last_undegraded":"2024-04-05T03:12:53.286319+0000","last_fullsized":"2024-04-05T03:12:53.286319+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:48:06.327074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271730+0000","last_change":"2024-04-05T03:10:31.866556+0000","last_active":"2024-04-05T03:12:53.271730+0000","last_peered":"2024-04-05T03:12:53.271730+0000","last_clean":"2024-04-05T03:12:53.271730+0000","last_became_active":"2024-04-05T03:10:31.865371+0000","last_became_peered":"2024-04-05T03:10:31.865371+0000","last_unstale":"2024-04-05T03:12:53.271730+0000","last_undegraded":"2024-04-05T03:12:53.271730+0000","last_fullsized":"2024-04-05T03:12:53.271730+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:48:34.182823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272547+0000","last_change":"2024-04-05T03:12:18.834658+0000","last_active":"2024-04-05T03:12:53.272547+0000","last_peered":"2024-04-05T03:12:53.272547+0000","last_clean":"2024-04-05T03:12:53.272547+0000","last_became_active":"2024-04-05T03:12:18.834340+0000","last_became_peered":"2024-04-05T03:12:18.834340+0000","last_unstale":"2024-04-05T03:12:53.272547+0000","last_undegraded":"2024-04-05T03:12:53.272547+0000","last_fullsized":"2024-04-05T03:12:53.272547+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:56:47.305361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"84'67","reported_seq":325,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.264221+0000","last_change":"2024-04-05T03:10:31.863085+0000","last_active":"2024-04-05T03:12:52.264221+0000","last_peered":"2024-04-05T03:12:52.264221+0000","last_clean":"2024-04-05T03:12:52.264221+0000","last_became_active":"2024-04-05T03:10:31.862321+0000","last_became_peered":"2024-04-05T03:10:31.862321+0000","last_unstale":"2024-04-05T03:12:52.264221+0000","last_undegraded":"2024-04-05T03:12:52.264221+0000","last_fullsized":"2024-04-05T03:12:52.264221+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:27:38.911052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028041299999999997,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":410,"num_read_kb":2404,"num_write":65,"num_write_kb":6191,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.272882+0000","last_change":"2024-04-05T03:12:18.831144+0000","last_active":"2024-04-05T03:12:53.272882+0000","last_peered":"2024-04-05T03:12:53.272882+0000","last_clean":"2024-04-05T03:12:53.272882+0000","last_became_active":"2024-04-05T03:12:18.830791+0000","last_became_peered":"2024-04-05T03:12:18.830791+0000","last_unstale":"2024-04-05T03:12:53.272882+0000","last_undegraded":"2024-04-05T03:12:53.272882+0000","last_fullsized":"2024-04-05T03:12:53.272882+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:54:42.730284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281338+0000","last_change":"2024-04-05T03:12:53.281338+0000","last_active":"2024-04-05T03:12:53.281338+0000","last_peered":"2024-04-05T03:12:53.281338+0000","last_clean":"2024-04-05T03:12:53.281338+0000","last_became_active":"2024-04-05T03:12:53.281008+0000","last_became_peered":"2024-04-05T03:12:53.281008+0000","last_unstale":"2024-04-05T03:12:53.281338+0000","last_undegraded":"2024-04-05T03:12:53.281338+0000","last_fullsized":"2024-04-05T03:12:53.281338+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:10:52.739314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'43","reported_seq":299,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266213+0000","last_change":"2024-04-05T03:11:44.821082+0000","last_active":"2024-04-05T03:12:52.266213+0000","last_peered":"2024-04-05T03:12:52.266213+0000","last_clean":"2024-04-05T03:12:52.266213+0000","last_became_active":"2024-04-05T03:11:44.820671+0000","last_became_peered":"2024-04-05T03:11:44.820671+0000","last_unstale":"2024-04-05T03:12:52.266213+0000","last_undegraded":"2024-04-05T03:12:52.266213+0000","last_fullsized":"2024-04-05T03:12:52.266213+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":43,"log_dups_size":0,"ondisk_log_size":43,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:40:01.852471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034224499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":329,"num_read_kb":305,"num_write":22,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.262555+0000","last_change":"2024-04-05T03:02:50.003073+0000","last_active":"2024-04-05T03:12:52.262555+0000","last_peered":"2024-04-05T03:12:52.262555+0000","last_clean":"2024-04-05T03:12:52.262555+0000","last_became_active":"2024-04-05T03:02:50.002726+0000","last_became_peered":"2024-04-05T03:02:50.002726+0000","last_unstale":"2024-04-05T03:12:52.262555+0000","last_undegraded":"2024-04-05T03:12:52.262555+0000","last_fullsized":"2024-04-05T03:12:52.262555+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:50:50.549731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284640+0000","last_change":"2024-04-05T03:12:53.284640+0000","last_active":"2024-04-05T03:12:53.284640+0000","last_peered":"2024-04-05T03:12:53.284640+0000","last_clean":"2024-04-05T03:12:53.284640+0000","last_became_active":"2024-04-05T03:12:53.283863+0000","last_became_peered":"2024-04-05T03:12:53.283863+0000","last_unstale":"2024-04-05T03:12:53.284640+0000","last_undegraded":"2024-04-05T03:12:53.284640+0000","last_fullsized":"2024-04-05T03:12:53.284640+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:08:49.107166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"88'161","reported_seq":449,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266125+0000","last_change":"2024-04-05T03:10:31.862851+0000","last_active":"2024-04-05T03:12:52.266125+0000","last_peered":"2024-04-05T03:12:52.266125+0000","last_clean":"2024-04-05T03:12:52.266125+0000","last_became_active":"2024-04-05T03:10:31.861865+0000","last_became_peered":"2024-04-05T03:10:31.861865+0000","last_unstale":"2024-04-05T03:12:52.266125+0000","last_undegraded":"2024-04-05T03:12:52.266125+0000","last_fullsized":"2024-04-05T03:12:52.266125+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:59:14.736708+0000","last_clean_scrub_stamp":"2024-04-05T02:59:14.736708+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:05:19.760756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"84'199","reported_seq":717,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258768+0000","last_change":"2024-04-05T03:12:18.830679+0000","last_active":"2024-04-05T03:12:52.258768+0000","last_peered":"2024-04-05T03:12:52.258768+0000","last_clean":"2024-04-05T03:12:52.258768+0000","last_became_active":"2024-04-05T03:12:18.830053+0000","last_became_peered":"2024-04-05T03:12:18.830053+0000","last_unstale":"2024-04-05T03:12:52.258768+0000","last_undegraded":"2024-04-05T03:12:52.258768+0000","last_fullsized":"2024-04-05T03:12:52.258768+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:41:46.843315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041847,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1018,"num_read_kb":1289,"num_write":171,"num_write_kb":1317,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271887+0000","last_change":"2024-04-05T03:11:44.819843+0000","last_active":"2024-04-05T03:12:53.271887+0000","last_peered":"2024-04-05T03:12:53.271887+0000","last_clean":"2024-04-05T03:12:53.271887+0000","last_became_active":"2024-04-05T03:11:44.819365+0000","last_became_peered":"2024-04-05T03:11:44.819365+0000","last_unstale":"2024-04-05T03:12:53.271887+0000","last_undegraded":"2024-04-05T03:12:53.271887+0000","last_fullsized":"2024-04-05T03:12:53.271887+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:04:22.836425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.270431+0000","last_change":"2024-04-05T03:12:18.831877+0000","last_active":"2024-04-05T03:12:53.270431+0000","last_peered":"2024-04-05T03:12:53.270431+0000","last_clean":"2024-04-05T03:12:53.270431+0000","last_became_active":"2024-04-05T03:12:18.831529+0000","last_became_peered":"2024-04-05T03:12:18.831529+0000","last_unstale":"2024-04-05T03:12:53.270431+0000","last_undegraded":"2024-04-05T03:12:53.270431+0000","last_fullsized":"2024-04-05T03:12:53.270431+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:11:32.246265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"84'150","reported_seq":693,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.279302+0000","last_change":"2024-04-05T03:12:53.279302+0000","last_active":"2024-04-05T03:12:53.279302+0000","last_peered":"2024-04-05T03:12:53.279302+0000","last_clean":"2024-04-05T03:12:53.279302+0000","last_became_active":"2024-04-05T03:12:53.278637+0000","last_became_peered":"2024-04-05T03:12:53.278637+0000","last_unstale":"2024-04-05T03:12:53.279302+0000","last_undegraded":"2024-04-05T03:12:53.279302+0000","last_fullsized":"2024-04-05T03:12:53.279302+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:02:40.892179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032387699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1041,"num_read_kb":3170,"num_write":130,"num_write_kb":2261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288343+0000","last_change":"2024-04-05T03:12:53.288056+0000","last_active":"2024-04-05T03:12:53.288343+0000","last_peered":"2024-04-05T03:12:53.288343+0000","last_clean":"2024-04-05T03:12:53.288343+0000","last_became_active":"2024-04-05T03:12:53.287593+0000","last_became_peered":"2024-04-05T03:12:53.287593+0000","last_unstale":"2024-04-05T03:12:53.288343+0000","last_undegraded":"2024-04-05T03:12:53.288343+0000","last_fullsized":"2024-04-05T03:12:53.288343+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:37:51.955764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265989+0000","last_change":"2024-04-05T03:10:31.868757+0000","last_active":"2024-04-05T03:12:52.265989+0000","last_peered":"2024-04-05T03:12:52.265989+0000","last_clean":"2024-04-05T03:12:52.265989+0000","last_became_active":"2024-04-05T03:10:31.868505+0000","last_became_peered":"2024-04-05T03:10:31.868505+0000","last_unstale":"2024-04-05T03:12:52.265989+0000","last_undegraded":"2024-04-05T03:12:52.265989+0000","last_fullsized":"2024-04-05T03:12:52.265989+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:32:37.077890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'125","reported_seq":793,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258739+0000","last_change":"2024-04-05T03:11:44.823007+0000","last_active":"2024-04-05T03:12:52.258739+0000","last_peered":"2024-04-05T03:12:52.258739+0000","last_clean":"2024-04-05T03:12:52.258739+0000","last_became_active":"2024-04-05T03:11:44.822476+0000","last_became_peered":"2024-04-05T03:11:44.822476+0000","last_unstale":"2024-04-05T03:12:52.258739+0000","last_undegraded":"2024-04-05T03:12:52.258739+0000","last_fullsized":"2024-04-05T03:12:52.258739+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:23:21.702987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053167100000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1152,"num_read_kb":1185,"num_write":104,"num_write_kb":138,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.270938+0000","last_change":"2024-04-05T03:11:44.824231+0000","last_active":"2024-04-05T03:12:53.270938+0000","last_peered":"2024-04-05T03:12:53.270938+0000","last_clean":"2024-04-05T03:12:53.270938+0000","last_became_active":"2024-04-05T03:11:44.823540+0000","last_became_peered":"2024-04-05T03:11:44.823540+0000","last_unstale":"2024-04-05T03:12:53.270938+0000","last_undegraded":"2024-04-05T03:12:53.270938+0000","last_fullsized":"2024-04-05T03:12:53.270938+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:47:47.664620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281466+0000","last_change":"2024-04-05T03:12:53.281466+0000","last_active":"2024-04-05T03:12:53.281466+0000","last_peered":"2024-04-05T03:12:53.281466+0000","last_clean":"2024-04-05T03:12:53.281466+0000","last_became_active":"2024-04-05T03:12:53.280200+0000","last_became_peered":"2024-04-05T03:12:53.280200+0000","last_unstale":"2024-04-05T03:12:53.281466+0000","last_undegraded":"2024-04-05T03:12:53.281466+0000","last_fullsized":"2024-04-05T03:12:53.281466+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:10:15.421941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"84'133","reported_seq":532,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271381+0000","last_change":"2024-04-05T03:11:44.822006+0000","last_active":"2024-04-05T03:12:53.271381+0000","last_peered":"2024-04-05T03:12:53.271381+0000","last_clean":"2024-04-05T03:12:53.271381+0000","last_became_active":"2024-04-05T03:11:44.821420+0000","last_became_peered":"2024-04-05T03:11:44.821420+0000","last_unstale":"2024-04-05T03:12:53.271381+0000","last_undegraded":"2024-04-05T03:12:53.271381+0000","last_fullsized":"2024-04-05T03:12:53.271381+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:01:42.643913+0000","last_clean_scrub_stamp":"2024-04-05T03:01:42.643913+0000","objects_scrubbed":0,"log_size":133,"log_dups_size":0,"ondisk_log_size":133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:40:41.011945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032491300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":648,"num_read_kb":1690,"num_write":135,"num_write_kb":1133,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284068+0000","last_change":"2024-04-05T03:12:53.284068+0000","last_active":"2024-04-05T03:12:53.284068+0000","last_peered":"2024-04-05T03:12:53.284068+0000","last_clean":"2024-04-05T03:12:53.284068+0000","last_became_active":"2024-04-05T03:12:53.283456+0000","last_became_peered":"2024-04-05T03:12:53.283456+0000","last_unstale":"2024-04-05T03:12:53.284068+0000","last_undegraded":"2024-04-05T03:12:53.284068+0000","last_fullsized":"2024-04-05T03:12:53.284068+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:31:16.994153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288318+0000","last_change":"2024-04-05T03:12:53.288318+0000","last_active":"2024-04-05T03:12:53.288318+0000","last_peered":"2024-04-05T03:12:53.288318+0000","last_clean":"2024-04-05T03:12:53.288318+0000","last_became_active":"2024-04-05T03:12:53.287961+0000","last_became_peered":"2024-04-05T03:12:53.287961+0000","last_unstale":"2024-04-05T03:12:53.288318+0000","last_undegraded":"2024-04-05T03:12:53.288318+0000","last_fullsized":"2024-04-05T03:12:53.288318+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:16:26.389764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266224+0000","last_change":"2024-04-05T03:11:44.824152+0000","last_active":"2024-04-05T03:12:52.266224+0000","last_peered":"2024-04-05T03:12:52.266224+0000","last_clean":"2024-04-05T03:12:52.266224+0000","last_became_active":"2024-04-05T03:11:44.823842+0000","last_became_peered":"2024-04-05T03:11:44.823842+0000","last_unstale":"2024-04-05T03:12:52.266224+0000","last_undegraded":"2024-04-05T03:12:52.266224+0000","last_fullsized":"2024-04-05T03:12:52.266224+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:26:31.940086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267009+0000","last_change":"2024-04-05T03:12:18.834468+0000","last_active":"2024-04-05T03:12:52.267009+0000","last_peered":"2024-04-05T03:12:52.267009+0000","last_clean":"2024-04-05T03:12:52.267009+0000","last_became_active":"2024-04-05T03:12:18.834140+0000","last_became_peered":"2024-04-05T03:12:18.834140+0000","last_unstale":"2024-04-05T03:12:52.267009+0000","last_undegraded":"2024-04-05T03:12:52.267009+0000","last_fullsized":"2024-04-05T03:12:52.267009+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:37:04.934029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266686+0000","last_change":"2024-04-05T03:12:18.834695+0000","last_active":"2024-04-05T03:12:52.266686+0000","last_peered":"2024-04-05T03:12:52.266686+0000","last_clean":"2024-04-05T03:12:52.266686+0000","last_became_active":"2024-04-05T03:12:18.834415+0000","last_became_peered":"2024-04-05T03:12:18.834415+0000","last_unstale":"2024-04-05T03:12:52.266686+0000","last_undegraded":"2024-04-05T03:12:52.266686+0000","last_fullsized":"2024-04-05T03:12:52.266686+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:07:47.977921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.263798+0000","last_change":"2024-04-05T03:11:44.829880+0000","last_active":"2024-04-05T03:12:52.263798+0000","last_peered":"2024-04-05T03:12:52.263798+0000","last_clean":"2024-04-05T03:12:52.263798+0000","last_became_active":"2024-04-05T03:11:44.829552+0000","last_became_peered":"2024-04-05T03:11:44.829552+0000","last_unstale":"2024-04-05T03:12:52.263798+0000","last_undegraded":"2024-04-05T03:12:52.263798+0000","last_fullsized":"2024-04-05T03:12:52.263798+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:36:37.307694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283661+0000","last_change":"2024-04-05T03:12:53.283661+0000","last_active":"2024-04-05T03:12:53.283661+0000","last_peered":"2024-04-05T03:12:53.283661+0000","last_clean":"2024-04-05T03:12:53.283661+0000","last_became_active":"2024-04-05T03:12:53.283271+0000","last_became_peered":"2024-04-05T03:12:53.283271+0000","last_unstale":"2024-04-05T03:12:53.283661+0000","last_undegraded":"2024-04-05T03:12:53.283661+0000","last_fullsized":"2024-04-05T03:12:53.283661+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:38:10.656186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288690+0000","last_change":"2024-04-05T03:12:53.288690+0000","last_active":"2024-04-05T03:12:53.288690+0000","last_peered":"2024-04-05T03:12:53.288690+0000","last_clean":"2024-04-05T03:12:53.288690+0000","last_became_active":"2024-04-05T03:12:53.287105+0000","last_became_peered":"2024-04-05T03:12:53.287105+0000","last_unstale":"2024-04-05T03:12:53.288690+0000","last_undegraded":"2024-04-05T03:12:53.288690+0000","last_fullsized":"2024-04-05T03:12:53.288690+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:45:04.692793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.271934+0000","last_change":"2024-04-05T03:11:44.822882+0000","last_active":"2024-04-05T03:12:53.271934+0000","last_peered":"2024-04-05T03:12:53.271934+0000","last_clean":"2024-04-05T03:12:53.271934+0000","last_became_active":"2024-04-05T03:11:44.822543+0000","last_became_peered":"2024-04-05T03:11:44.822543+0000","last_unstale":"2024-04-05T03:12:53.271934+0000","last_undegraded":"2024-04-05T03:12:53.271934+0000","last_fullsized":"2024-04-05T03:12:53.271934+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:22:12.438776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284191+0000","last_change":"2024-04-05T03:12:53.284191+0000","last_active":"2024-04-05T03:12:53.284191+0000","last_peered":"2024-04-05T03:12:53.284191+0000","last_clean":"2024-04-05T03:12:53.284191+0000","last_became_active":"2024-04-05T03:12:53.283859+0000","last_became_peered":"2024-04-05T03:12:53.283859+0000","last_unstale":"2024-04-05T03:12:53.284191+0000","last_undegraded":"2024-04-05T03:12:53.284191+0000","last_fullsized":"2024-04-05T03:12:53.284191+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:49:28.905998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.263640+0000","last_change":"2024-04-05T03:12:18.831147+0000","last_active":"2024-04-05T03:12:52.263640+0000","last_peered":"2024-04-05T03:12:52.263640+0000","last_clean":"2024-04-05T03:12:52.263640+0000","last_became_active":"2024-04-05T03:12:18.830779+0000","last_became_peered":"2024-04-05T03:12:18.830779+0000","last_unstale":"2024-04-05T03:12:52.263640+0000","last_undegraded":"2024-04-05T03:12:52.263640+0000","last_fullsized":"2024-04-05T03:12:52.263640+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:18:18.393332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":53,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.264337+0000","last_change":"2024-04-05T03:02:48.461252+0000","last_active":"2024-04-05T03:12:52.264337+0000","last_peered":"2024-04-05T03:12:52.264337+0000","last_clean":"2024-04-05T03:12:52.264337+0000","last_became_active":"2024-04-05T03:02:48.460982+0000","last_became_peered":"2024-04-05T03:02:48.460982+0000","last_unstale":"2024-04-05T03:12:52.264337+0000","last_undegraded":"2024-04-05T03:12:52.264337+0000","last_fullsized":"2024-04-05T03:12:52.264337+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:03.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286911+0000","last_change":"2024-04-05T03:12:53.286911+0000","last_active":"2024-04-05T03:12:53.286911+0000","last_peered":"2024-04-05T03:12:53.286911+0000","last_clean":"2024-04-05T03:12:53.286911+0000","last_became_active":"2024-04-05T03:12:53.286680+0000","last_became_peered":"2024-04-05T03:12:53.286680+0000","last_unstale":"2024-04-05T03:12:53.286911+0000","last_undegraded":"2024-04-05T03:12:53.286911+0000","last_fullsized":"2024-04-05T03:12:53.286911+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:04:27.036557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":71,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.264346+0000","last_change":"2024-04-05T03:11:44.822325+0000","last_active":"2024-04-05T03:12:52.264346+0000","last_peered":"2024-04-05T03:12:52.264346+0000","last_clean":"2024-04-05T03:12:52.264346+0000","last_became_active":"2024-04-05T03:11:44.821851+0000","last_became_peered":"2024-04-05T03:11:44.821851+0000","last_unstale":"2024-04-05T03:12:52.264346+0000","last_undegraded":"2024-04-05T03:12:52.264346+0000","last_fullsized":"2024-04-05T03:12:52.264346+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:34:03.316393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258716+0000","last_change":"2024-04-05T03:02:48.903134+0000","last_active":"2024-04-05T03:12:52.258716+0000","last_peered":"2024-04-05T03:12:52.258716+0000","last_clean":"2024-04-05T03:12:52.258716+0000","last_became_active":"2024-04-05T03:02:48.902836+0000","last_became_peered":"2024-04-05T03:02:48.902836+0000","last_unstale":"2024-04-05T03:12:52.258716+0000","last_undegraded":"2024-04-05T03:12:52.258716+0000","last_fullsized":"2024-04-05T03:12:52.258716+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:08:54.250101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":70,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265091+0000","last_change":"2024-04-05T03:10:31.862564+0000","last_active":"2024-04-05T03:12:52.265091+0000","last_peered":"2024-04-05T03:12:52.265091+0000","last_clean":"2024-04-05T03:12:52.265091+0000","last_became_active":"2024-04-05T03:10:31.862202+0000","last_became_peered":"2024-04-05T03:10:31.862202+0000","last_unstale":"2024-04-05T03:12:52.265091+0000","last_undegraded":"2024-04-05T03:12:52.265091+0000","last_fullsized":"2024-04-05T03:12:52.265091+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:34:22.600326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.289926+0000","last_change":"2024-04-05T03:12:53.289926+0000","last_active":"2024-04-05T03:12:53.289926+0000","last_peered":"2024-04-05T03:12:53.289926+0000","last_clean":"2024-04-05T03:12:53.289926+0000","last_became_active":"2024-04-05T03:12:53.288900+0000","last_became_peered":"2024-04-05T03:12:53.288900+0000","last_unstale":"2024-04-05T03:12:53.289926+0000","last_undegraded":"2024-04-05T03:12:53.289926+0000","last_fullsized":"2024-04-05T03:12:53.289926+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:21:50.597917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":77,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.261525+0000","last_change":"2024-04-05T03:10:31.860733+0000","last_active":"2024-04-05T03:12:52.261525+0000","last_peered":"2024-04-05T03:12:52.261525+0000","last_clean":"2024-04-05T03:12:52.261525+0000","last_became_active":"2024-04-05T03:10:31.860273+0000","last_became_peered":"2024-04-05T03:10:31.860273+0000","last_unstale":"2024-04-05T03:12:52.261525+0000","last_undegraded":"2024-04-05T03:12:52.261525+0000","last_fullsized":"2024-04-05T03:12:52.261525+0000","mapping_epoch":92,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":93,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:05:04.356023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283189+0000","last_change":"2024-04-05T03:12:53.282806+0000","last_active":"2024-04-05T03:12:53.283189+0000","last_peered":"2024-04-05T03:12:53.283189+0000","last_clean":"2024-04-05T03:12:53.283189+0000","last_became_active":"2024-04-05T03:12:53.282476+0000","last_became_peered":"2024-04-05T03:12:53.282476+0000","last_unstale":"2024-04-05T03:12:53.283189+0000","last_undegraded":"2024-04-05T03:12:53.283189+0000","last_fullsized":"2024-04-05T03:12:53.283189+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:20:50.423294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.263585+0000","last_change":"2024-04-05T03:02:48.460853+0000","last_active":"2024-04-05T03:12:52.263585+0000","last_peered":"2024-04-05T03:12:52.263585+0000","last_clean":"2024-04-05T03:12:52.263585+0000","last_became_active":"2024-04-05T03:02:48.460557+0000","last_became_peered":"2024-04-05T03:02:48.460557+0000","last_unstale":"2024-04-05T03:12:52.263585+0000","last_undegraded":"2024-04-05T03:12:52.263585+0000","last_fullsized":"2024-04-05T03:12:52.263585+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:57.261993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281677+0000","last_change":"2024-04-05T03:12:53.281677+0000","last_active":"2024-04-05T03:12:53.281677+0000","last_peered":"2024-04-05T03:12:53.281677+0000","last_clean":"2024-04-05T03:12:53.281677+0000","last_became_active":"2024-04-05T03:12:53.280903+0000","last_became_peered":"2024-04-05T03:12:53.280903+0000","last_unstale":"2024-04-05T03:12:53.281677+0000","last_undegraded":"2024-04-05T03:12:53.281677+0000","last_fullsized":"2024-04-05T03:12:53.281677+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:56:09.930244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281646+0000","last_change":"2024-04-05T03:12:53.281646+0000","last_active":"2024-04-05T03:12:53.281646+0000","last_peered":"2024-04-05T03:12:53.281646+0000","last_clean":"2024-04-05T03:12:53.281646+0000","last_became_active":"2024-04-05T03:12:53.281333+0000","last_became_peered":"2024-04-05T03:12:53.281333+0000","last_unstale":"2024-04-05T03:12:53.281646+0000","last_undegraded":"2024-04-05T03:12:53.281646+0000","last_fullsized":"2024-04-05T03:12:53.281646+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:20:08.398538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.289627+0000","last_change":"2024-04-05T03:12:53.289627+0000","last_active":"2024-04-05T03:12:53.289627+0000","last_peered":"2024-04-05T03:12:53.289627+0000","last_clean":"2024-04-05T03:12:53.289627+0000","last_became_active":"2024-04-05T03:12:53.289111+0000","last_became_peered":"2024-04-05T03:12:53.289111+0000","last_unstale":"2024-04-05T03:12:53.289627+0000","last_undegraded":"2024-04-05T03:12:53.289627+0000","last_fullsized":"2024-04-05T03:12:53.289627+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:56:31.406641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":71,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.262796+0000","last_change":"2024-04-05T03:12:18.833440+0000","last_active":"2024-04-05T03:12:52.262796+0000","last_peered":"2024-04-05T03:12:52.262796+0000","last_clean":"2024-04-05T03:12:52.262796+0000","last_became_active":"2024-04-05T03:12:18.833168+0000","last_became_peered":"2024-04-05T03:12:18.833168+0000","last_unstale":"2024-04-05T03:12:52.262796+0000","last_undegraded":"2024-04-05T03:12:52.262796+0000","last_fullsized":"2024-04-05T03:12:52.262796+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:27:36.928082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.266296+0000","last_change":"2024-04-05T03:11:44.827920+0000","last_active":"2024-04-05T03:12:52.266296+0000","last_peered":"2024-04-05T03:12:52.266296+0000","last_clean":"2024-04-05T03:12:52.266296+0000","last_became_active":"2024-04-05T03:11:44.827537+0000","last_became_peered":"2024-04-05T03:11:44.827537+0000","last_unstale":"2024-04-05T03:12:52.266296+0000","last_undegraded":"2024-04-05T03:12:52.266296+0000","last_fullsized":"2024-04-05T03:12:52.266296+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:59:25.074973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"88'22","reported_seq":74,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267978+0000","last_change":"2024-04-05T03:02:48.455833+0000","last_active":"2024-04-05T03:12:52.267978+0000","last_peered":"2024-04-05T03:12:52.267978+0000","last_clean":"2024-04-05T03:12:52.267978+0000","last_became_active":"2024-04-05T03:02:48.455185+0000","last_became_peered":"2024-04-05T03:02:48.455185+0000","last_unstale":"2024-04-05T03:12:52.267978+0000","last_undegraded":"2024-04-05T03:12:52.267978+0000","last_fullsized":"2024-04-05T03:12:52.267978+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:02.621579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.270987+0000","last_change":"2024-04-05T03:12:18.831110+0000","last_active":"2024-04-05T03:12:53.270987+0000","last_peered":"2024-04-05T03:12:53.270987+0000","last_clean":"2024-04-05T03:12:53.270987+0000","last_became_active":"2024-04-05T03:12:18.830763+0000","last_became_peered":"2024-04-05T03:12:18.830763+0000","last_unstale":"2024-04-05T03:12:53.270987+0000","last_undegraded":"2024-04-05T03:12:53.270987+0000","last_fullsized":"2024-04-05T03:12:53.270987+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:07:13.205635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288134+0000","last_change":"2024-04-05T03:12:53.288134+0000","last_active":"2024-04-05T03:12:53.288134+0000","last_peered":"2024-04-05T03:12:53.288134+0000","last_clean":"2024-04-05T03:12:53.288134+0000","last_became_active":"2024-04-05T03:12:53.287780+0000","last_became_peered":"2024-04-05T03:12:53.287780+0000","last_unstale":"2024-04-05T03:12:53.288134+0000","last_undegraded":"2024-04-05T03:12:53.288134+0000","last_fullsized":"2024-04-05T03:12:53.288134+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:48:29.253792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.286200+0000","last_change":"2024-04-05T03:12:53.286200+0000","last_active":"2024-04-05T03:12:53.286200+0000","last_peered":"2024-04-05T03:12:53.286200+0000","last_clean":"2024-04-05T03:12:53.286200+0000","last_became_active":"2024-04-05T03:12:53.285836+0000","last_became_peered":"2024-04-05T03:12:53.285836+0000","last_unstale":"2024-04-05T03:12:53.286200+0000","last_undegraded":"2024-04-05T03:12:53.286200+0000","last_fullsized":"2024-04-05T03:12:53.286200+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:32:39.977616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.258955+0000","last_change":"2024-04-05T03:11:44.821345+0000","last_active":"2024-04-05T03:12:52.258955+0000","last_peered":"2024-04-05T03:12:52.258955+0000","last_clean":"2024-04-05T03:12:52.258955+0000","last_became_active":"2024-04-05T03:11:44.821003+0000","last_became_peered":"2024-04-05T03:11:44.821003+0000","last_unstale":"2024-04-05T03:12:52.258955+0000","last_undegraded":"2024-04-05T03:12:52.258955+0000","last_fullsized":"2024-04-05T03:12:52.258955+0000","mapping_epoch":97,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":98,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:08:49.149318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.289139+0000","last_change":"2024-04-05T03:12:53.289139+0000","last_active":"2024-04-05T03:12:53.289139+0000","last_peered":"2024-04-05T03:12:53.289139+0000","last_clean":"2024-04-05T03:12:53.289139+0000","last_became_active":"2024-04-05T03:12:53.288830+0000","last_became_peered":"2024-04-05T03:12:53.288830+0000","last_unstale":"2024-04-05T03:12:53.289139+0000","last_undegraded":"2024-04-05T03:12:53.289139+0000","last_fullsized":"2024-04-05T03:12:53.289139+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:15:45.481068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.281901+0000","last_change":"2024-04-05T03:12:53.281302+0000","last_active":"2024-04-05T03:12:53.281901+0000","last_peered":"2024-04-05T03:12:53.281901+0000","last_clean":"2024-04-05T03:12:53.281901+0000","last_became_active":"2024-04-05T03:12:53.280958+0000","last_became_peered":"2024-04-05T03:12:53.280958+0000","last_unstale":"2024-04-05T03:12:53.281901+0000","last_undegraded":"2024-04-05T03:12:53.281901+0000","last_fullsized":"2024-04-05T03:12:53.281901+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:35:26.995968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.267501+0000","last_change":"2024-04-05T03:12:18.835520+0000","last_active":"2024-04-05T03:12:52.267501+0000","last_peered":"2024-04-05T03:12:52.267501+0000","last_clean":"2024-04-05T03:12:52.267501+0000","last_became_active":"2024-04-05T03:12:18.834156+0000","last_became_peered":"2024-04-05T03:12:18.834156+0000","last_unstale":"2024-04-05T03:12:52.267501+0000","last_undegraded":"2024-04-05T03:12:52.267501+0000","last_fullsized":"2024-04-05T03:12:52.267501+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:36:18.106186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.288550+0000","last_change":"2024-04-05T03:12:53.288550+0000","last_active":"2024-04-05T03:12:53.288550+0000","last_peered":"2024-04-05T03:12:53.288550+0000","last_clean":"2024-04-05T03:12:53.288550+0000","last_became_active":"2024-04-05T03:12:53.288240+0000","last_became_peered":"2024-04-05T03:12:53.288240+0000","last_unstale":"2024-04-05T03:12:53.288550+0000","last_undegraded":"2024-04-05T03:12:53.288550+0000","last_fullsized":"2024-04-05T03:12:53.288550+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:09:23.144915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.284139+0000","last_change":"2024-04-05T03:12:53.284139+0000","last_active":"2024-04-05T03:12:53.284139+0000","last_peered":"2024-04-05T03:12:53.284139+0000","last_clean":"2024-04-05T03:12:53.284139+0000","last_became_active":"2024-04-05T03:12:53.283489+0000","last_became_peered":"2024-04-05T03:12:53.283489+0000","last_unstale":"2024-04-05T03:12:53.284139+0000","last_undegraded":"2024-04-05T03:12:53.284139+0000","last_fullsized":"2024-04-05T03:12:53.284139+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:55:01.202431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.283162+0000","last_change":"2024-04-05T03:12:53.283162+0000","last_active":"2024-04-05T03:12:53.283162+0000","last_peered":"2024-04-05T03:12:53.283162+0000","last_clean":"2024-04-05T03:12:53.283162+0000","last_became_active":"2024-04-05T03:12:53.282796+0000","last_became_peered":"2024-04-05T03:12:53.282796+0000","last_unstale":"2024-04-05T03:12:53.283162+0000","last_undegraded":"2024-04-05T03:12:53.283162+0000","last_fullsized":"2024-04-05T03:12:53.283162+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:44:35.054496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":36,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-04-05T03:12:53.282737+0000","last_change":"2024-04-05T03:12:53.282737+0000","last_active":"2024-04-05T03:12:53.282737+0000","last_peered":"2024-04-05T03:12:53.282737+0000","last_clean":"2024-04-05T03:12:53.282737+0000","last_became_active":"2024-04-05T03:12:53.281638+0000","last_became_peered":"2024-04-05T03:12:53.281638+0000","last_unstale":"2024-04-05T03:12:53.282737+0000","last_undegraded":"2024-04-05T03:12:53.282737+0000","last_fullsized":"2024-04-05T03:12:53.282737+0000","mapping_epoch":107,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":108,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:48.434389+0000","last_clean_scrub_stamp":"2024-04-05T03:02:48.434389+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:37.735394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":87,"reported_epoch":107,"state":"active+clean","last_fresh":"2024-04-05T03:12:52.265893+0000","last_change":"2024-04-05T03:12:18.834654+0000","last_active":"2024-04-05T03:12:52.265893+0000","last_peered":"2024-04-05T03:12:52.265893+0000","last_clean":"2024-04-05T03:12:52.265893+0000","last_became_active":"2024-04-05T03:12:18.834305+0000","last_became_peered":"2024-04-05T03:12:18.834305+0000","last_unstale":"2024-04-05T03:12:52.265893+0000","last_undegraded":"2024-04-05T03:12:52.265893+0000","last_fullsized":"2024-04-05T03:12:52.265893+0000","mapping_epoch":102,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":103,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T03:02:47.430803+0000","last_clean_scrub_stamp":"2024-04-05T03:02:47.430803+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:13:51.558689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18023,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":12992,"num_write":846,"num_write_kb":12586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5428,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201436,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307224,"kb_used_data":5772,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93437928,"statfs":{"total":95995035648,"available":95680438272,"internally_reserved":0,"allocated":5910528,"data_stored":5387602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463073,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303120,"kb_used_data":1664,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93442032,"statfs":{"total":95995035648,"available":95684640768,"internally_reserved":0,"allocated":1703936,"data_stored":1192927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":97,"seq":416611827740,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32824,"kb_used_data":1124,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93712328,"statfs":{"total":95995035648,"available":95961423872,"internally_reserved":0,"allocated":1150976,"data_stored":608317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8054,"internal_metadata":32432266},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":92,"seq":395136991274,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37508,"kb_used_data":5816,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93707644,"statfs":{"total":95995035648,"available":95956627456,"internally_reserved":0,"allocated":5955584,"data_stored":5394175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7741,"internal_metadata":32432579},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":102,"seq":438086664213,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1112,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1138688,"data_stored":605169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8171,"internal_metadata":28172309},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":107,"seq":459561500686,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28632,"kb_used_data":1092,"kb_used_omap":6,"kb_used_meta":27513,"kb_avail":93716520,"statfs":{"total":95995035648,"available":95965716480,"internally_reserved":0,"allocated":1118208,"data_stored":602088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6301,"internal_metadata":28174179},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986346,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302544,"kb_used_data":5188,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442608,"statfs":{"total":95995035648,"available":95685230592,"internally_reserved":0,"allocated":5312512,"data_stored":4797164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724709,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298440,"kb_used_data":1080,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446712,"statfs":{"total":95995035648,"available":95689433088,"internally_reserved":0,"allocated":1105920,"data_stored":598629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3845,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1583,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6037,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6350,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2622,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3014,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:13:54.867 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-05T03:13:54.867 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-05T03:13:54.867 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-05T03:13:54.867 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:55.136 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:55.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:55 smithi033 ceph-mon[101783]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:13:55.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:55 smithi033 ceph-mon[97738]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:13:55.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:55 smithi151 ceph-mon[88672]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:13:56.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:56 smithi033 ceph-mon[97738]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:56.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:56 smithi033 ceph-mon[101783]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:56.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:56 smithi151 ceph-mon[88672]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:57.032 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:13:57.033 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:57.357 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:57 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1986793202' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:57.357 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:57 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1986793202' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:57.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:57 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1986793202' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:58.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:13:58 smithi151 ceph-mon[88672]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:58.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:13:58 smithi033 ceph-mon[97738]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:13:58.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:13:58 smithi033 ceph-mon[101783]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:00.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:00 smithi151 ceph-mon[88672]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:00.532 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:00 smithi033 ceph-mon[97738]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:00.533 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:00 smithi033 ceph-mon[101783]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:00.747 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:00.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:14:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:14:00] "GET /metrics HTTP/1.1" 200 34221 "" "Prometheus/2.43.0" 2024-04-05T03:14:01.030 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:02.277 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:02 smithi151 ceph-mon[88672]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:02.278 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:02.358 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:02 smithi033 ceph-mon[101783]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:02.359 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:02.359 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:02 smithi033 ceph-mon[97738]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:02.359 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:02.961 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:02.961 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:03.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:03 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1877893954' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:03.548 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:03 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1877893954' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:03.548 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:03 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1877893954' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:04.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:04 smithi151 ceph-mon[88672]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:04.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:04 smithi033 ceph-mon[97738]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:04.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:04 smithi033 ceph-mon[101783]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:06.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:06 smithi151 ceph-mon[88672]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:06.575 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:06.605 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:06 smithi033 ceph-mon[101783]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:06.605 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:06 smithi033 ceph-mon[97738]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:06.845 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:08.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:08 smithi033 ceph-mon[97738]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:08.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:08 smithi033 ceph-mon[101783]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:08.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:08 smithi151 ceph-mon[88672]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:08.780 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:08.780 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:09.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:09 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3218497165' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:09.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:09 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3218497165' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:09.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:09 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3218497165' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:10.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:10 smithi033 ceph-mon[97738]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:10.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:10 smithi033 ceph-mon[101783]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:10.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:10 smithi151 ceph-mon[88672]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:11.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:14:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:14:10] "GET /metrics HTTP/1.1" 200 34221 "" "Prometheus/2.43.0" 2024-04-05T03:14:11.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:11 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:14:11.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:11 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:14:11.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:11 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:14:12.415 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:12.713 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:12.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:12 smithi033 ceph-mon[101783]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:12.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:12 smithi033 ceph-mon[97738]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:12.776 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:12 smithi151 ceph-mon[88672]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:14.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:14 smithi033 ceph-mon[97738]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:14.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:14 smithi033 ceph-mon[101783]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:14.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:14 smithi151 ceph-mon[88672]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:15.092 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:15.093 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:16.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:16.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:16.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2973548914' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:16.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[101783]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:16.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:16.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:16.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2973548914' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:16.231 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:16 smithi033 ceph-mon[97738]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:16.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:16 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:16.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:16 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:16.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:16 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2973548914' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:16.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:16 smithi151 ceph-mon[88672]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:17.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:17.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:17.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:17.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:17.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:18.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:18.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[101783]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:18.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:14:18.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:14:18.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:18.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:18.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[97738]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:18.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:14:18.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:14:18.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:18.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:18.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:18 smithi151 ceph-mon[88672]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:18.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:14:18.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:14:18.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:14:18.887 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:19.160 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:19 smithi033 ceph-mon[97738]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:19.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:19 smithi033 ceph-mon[97738]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:19.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:19 smithi033 ceph-mon[101783]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:19.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:19 smithi033 ceph-mon[101783]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:19.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:19 smithi151 ceph-mon[88672]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:19.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:19 smithi151 ceph-mon[88672]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:20.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:14:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:14:20] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2024-04-05T03:14:21.132 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:21.132 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:21.329 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:21 smithi151 ceph-mon[88672]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:21.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:21 smithi033 ceph-mon[97738]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:21.466 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:21 smithi033 ceph-mon[101783]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:22.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:22 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1561551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:22.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:22 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1561551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:22.528 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:22 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1561551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:23.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:23 smithi033 ceph-mon[101783]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:23.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:23 smithi033 ceph-mon[97738]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:23.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:23 smithi151 ceph-mon[88672]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:24.777 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:25.053 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:25.385 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:25 smithi033 ceph-mon[97738]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:25.386 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:25 smithi033 ceph-mon[101783]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:25.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:25 smithi151 ceph-mon[88672]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:26.966 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:26.966 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:27.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:27 smithi033 ceph-mon[97738]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:27.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:27 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3717925910' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:27 smithi033 ceph-mon[101783]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:27.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:27 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3717925910' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:27.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:27 smithi151 ceph-mon[88672]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:27.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:27 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3717925910' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:29.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:29 smithi033 ceph-mon[97738]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:29.481 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:29 smithi033 ceph-mon[101783]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:29.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:29 smithi151 ceph-mon[88672]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:30.633 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:30.907 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:30.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:14:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:14:30] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2024-04-05T03:14:31.243 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:31 smithi033 ceph-mon[101783]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:31.244 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:31 smithi033 ceph-mon[97738]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:31.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:31 smithi151 ceph-mon[88672]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:32.293 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:32.293 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:32.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:32.970 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:32.970 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:33.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:33 smithi033 ceph-mon[101783]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:33.230 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:33 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3693289145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:33.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:33 smithi033 ceph-mon[97738]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:33.230 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:33 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3693289145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:33.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:33 smithi151 ceph-mon[88672]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:33.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:33 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3693289145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:35.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:35 smithi033 ceph-mon[97738]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:35.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:35 smithi033 ceph-mon[101783]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:35.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:35 smithi151 ceph-mon[88672]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:36.649 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:36.927 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:37.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:37 smithi033 ceph-mon[97738]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:37 smithi033 ceph-mon[101783]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:37.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:37 smithi151 ceph-mon[88672]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:38.771 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:38.771 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:39.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:39 smithi033 ceph-mon[97738]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:39.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:39 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1692997455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:39.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:39 smithi033 ceph-mon[101783]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:39.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:39 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1692997455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:39.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:39 smithi151 ceph-mon[88672]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:39.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:39 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1692997455' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:14:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:14:40] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2024-04-05T03:14:41.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:41 smithi033 ceph-mon[97738]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:41 smithi033 ceph-mon[101783]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:41.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:41 smithi151 ceph-mon[88672]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:42.448 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:42.722 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:43 smithi033 ceph-mon[97738]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:43.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:43 smithi033 ceph-mon[101783]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:43.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:43 smithi151 ceph-mon[88672]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:44.534 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:44.534 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:45.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:45 smithi033 ceph-mon[97738]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:45.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:45 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1786905253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:45.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:45 smithi033 ceph-mon[101783]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:45.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:45 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1786905253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:45.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:45 smithi151 ceph-mon[88672]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:45.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:45 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1786905253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:47 smithi033 ceph-mon[97738]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:47 smithi033 ceph-mon[101783]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:47.731 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:47.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:47 smithi151 ceph-mon[88672]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:47.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:48.153 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:48.429 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:49.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:49 smithi033 ceph-mon[97738]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:49.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:49 smithi033 ceph-mon[101783]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:49.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:49 smithi151 ceph-mon[88672]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:50.231 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:50.231 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:50.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:50 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/250534485' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:50.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:50 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/250534485' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:50.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:50 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/250534485' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:51.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:14:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:14:50] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2024-04-05T03:14:51.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:51 smithi151 ceph-mon[88672]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:51.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:51 smithi033 ceph-mon[97738]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:51.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:51 smithi033 ceph-mon[101783]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:53.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:53 smithi151 ceph-mon[88672]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:53.833 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:53.864 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:53 smithi033 ceph-mon[97738]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:53.864 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:53 smithi033 ceph-mon[101783]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:54.106 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:55.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:55 smithi151 ceph-mon[88672]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:55.857 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:55 smithi033 ceph-mon[101783]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:55.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:55 smithi033 ceph-mon[97738]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:55.996 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:14:55.996 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:56.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:56 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3970207867' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:56 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3970207867' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:56.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:56 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3970207867' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:57.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:57 smithi151 ceph-mon[88672]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:57.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:57 smithi033 ceph-mon[101783]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:57.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:57 smithi033 ceph-mon[97738]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:59.622 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:59.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:14:59 smithi151 ceph-mon[88672]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:59.894 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:59.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:14:59 smithi033 ceph-mon[97738]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:14:59.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:14:59 smithi033 ceph-mon[101783]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:01.035 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:15:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:15:00] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2024-04-05T03:15:01.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:01 smithi033 ceph-mon[97738]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:01.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:01 smithi033 ceph-mon[101783]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:01.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:01 smithi151 ceph-mon[88672]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:01.845 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:01.845 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:02.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:02.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:02 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2411812609' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:02.847 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:02.847 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:02 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2411812609' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:02.847 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:02.847 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:02 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2411812609' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:03.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:03 smithi151 ceph-mon[88672]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:03.805 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:03 smithi033 ceph-mon[97738]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:03.805 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:03 smithi033 ceph-mon[101783]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:05.582 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:05.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:05 smithi151 ceph-mon[88672]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:05.857 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:05.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:05 smithi033 ceph-mon[97738]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:05.979 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:05 smithi033 ceph-mon[101783]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:07.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:07 smithi033 ceph-mon[97738]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:07.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:07 smithi033 ceph-mon[101783]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:07.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:07 smithi151 ceph-mon[88672]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:07.779 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:07.779 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:08.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:08 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2570724273' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:08.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:08 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2570724273' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:08.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:08 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2570724273' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:09.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:09 smithi033 ceph-mon[97738]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:09.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:09 smithi033 ceph-mon[101783]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:10.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:09 smithi151 ceph-mon[88672]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:11.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:15:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:15:10] "GET /metrics HTTP/1.1" 200 34218 "" "Prometheus/2.43.0" 2024-04-05T03:15:11.485 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:11.758 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:11.881 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:11 smithi033 ceph-mon[97738]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:11.881 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:11 smithi033 ceph-mon[101783]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:12.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:11 smithi151 ceph-mon[88672]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:13.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:13 smithi033 ceph-mon[97738]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:13.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:13 smithi033 ceph-mon[101783]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:13.773 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:13.773 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:14.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:13 smithi151 ceph-mon[88672]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:14.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:14 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3756619037' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:14.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:14 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3756619037' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:15.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:14 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3756619037' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:15.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:15 smithi033 ceph-mon[97738]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:15.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:15 smithi033 ceph-mon[101783]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:16.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:15 smithi151 ceph-mon[88672]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:17.496 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:17.780 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:17.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:17 smithi033 ceph-mon[97738]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:17.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:17.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:17 smithi033 ceph-mon[101783]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:17.904 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:18.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:17 smithi151 ceph-mon[88672]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:18.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:18.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:18 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:15:18.917 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:18 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:15:18.917 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:18 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:15:19.857 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:19.858 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:19.858 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:19 smithi033 ceph-mon[101783]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:19.858 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:19 smithi033 ceph-mon[97738]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:20.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:19 smithi151 ceph-mon[88672]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:20.808 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:15:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:15:20] "GET /metrics HTTP/1.1" 200 34218 "" "Prometheus/2.43.0" 2024-04-05T03:15:20.808 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:20 smithi033 ceph-mon[101783]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:20.808 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:20 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1672960893' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:20.808 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:20 smithi033 ceph-mon[97738]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:20.808 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:20 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1672960893' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:20.942 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:20 smithi151 ceph-mon[88672]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:20.942 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:20 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1672960893' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:22.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:22.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:22 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:22 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:22 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:23.569 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:23.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:23 smithi033 ceph-mon[101783]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:23.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:23 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:23.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:23 smithi033 ceph-mon[97738]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:23.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:23 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:23.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:23 smithi151 ceph-mon[88672]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:23.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:23 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:23.841 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:24.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:24 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:24.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:24 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:24.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:24 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:25.724 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:25.724 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:25.724 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[97738]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:25.724 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:15:25.724 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:15:25.724 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:25.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[101783]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:25.725 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:15:25.725 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:15:25.725 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:25 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:25.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:25 smithi151 ceph-mon[88672]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:25.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:25 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:15:25.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:25 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:15:25.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:25 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:15:26.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:26 smithi033 ceph-mon[101783]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:26.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:26 smithi033 ceph-mon[101783]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:26.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:26 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2702988546' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:26 smithi033 ceph-mon[97738]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:26 smithi033 ceph-mon[97738]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:26 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2702988546' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:26.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:26 smithi151 ceph-mon[88672]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:26.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:26 smithi151 ceph-mon[88672]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:26.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:26 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2702988546' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:28.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:28 smithi151 ceph-mon[88672]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:28.979 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:28 smithi033 ceph-mon[101783]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:28.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:28 smithi033 ceph-mon[97738]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:29.290 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:29.563 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:30.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:30 smithi033 ceph-mon[97738]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:30.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:30 smithi033 ceph-mon[101783]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:30.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:30 smithi151 ceph-mon[88672]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:31.103 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:15:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:15:30] "GET /metrics HTTP/1.1" 200 34218 "" "Prometheus/2.43.0" 2024-04-05T03:15:31.609 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:31.609 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:32.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:32 smithi033 ceph-mon[97738]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:32.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:32 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3628080892' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:32.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:32.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:32 smithi151 ceph-mon[88672]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:32.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:32 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3628080892' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:32.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:32.979 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:32 smithi033 ceph-mon[101783]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:32.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:32 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3628080892' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:32.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:34.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:34 smithi151 ceph-mon[88672]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:34.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:34 smithi033 ceph-mon[97738]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:34.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:34 smithi033 ceph-mon[101783]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:35.271 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:35.548 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:36.670 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:36 smithi033 ceph-mon[97738]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:36.670 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:36 smithi033 ceph-mon[101783]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:36.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:36 smithi151 ceph-mon[88672]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:37.432 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:37.432 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:37.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:37 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3738509707' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:37.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:37 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3738509707' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:37.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:37 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3738509707' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:38.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:38 smithi151 ceph-mon[88672]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:38.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:38 smithi033 ceph-mon[97738]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:38.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:38 smithi033 ceph-mon[101783]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:40.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:40 smithi151 ceph-mon[88672]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:40.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:40 smithi033 ceph-mon[97738]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:40.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:40 smithi033 ceph-mon[101783]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:40.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:15:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:15:40] "GET /metrics HTTP/1.1" 200 34221 "" "Prometheus/2.43.0" 2024-04-05T03:15:41.026 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:41.300 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:42.661 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:42 smithi033 ceph-mon[97738]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:42.662 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:42 smithi033 ceph-mon[101783]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:42.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:42 smithi151 ceph-mon[88672]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:43.147 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:43.147 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:43.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:43 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3391908004' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:43.785 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:43 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3391908004' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:43.785 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:43 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3391908004' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:44.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:44 smithi151 ceph-mon[88672]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:44.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:44 smithi033 ceph-mon[97738]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:44.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:44 smithi033 ceph-mon[101783]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:46.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:46 smithi151 ceph-mon[88672]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:46.809 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:46.839 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:46 smithi033 ceph-mon[97738]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:46.839 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:46 smithi033 ceph-mon[101783]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:47.083 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:47.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:47.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:48.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:48 smithi033 ceph-mon[97738]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:48.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:48 smithi033 ceph-mon[101783]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:48.861 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:48.862 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:49.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:48 smithi151 ceph-mon[88672]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:49.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:49 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1292068589' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:49.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:49 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1292068589' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:50.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:49 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1292068589' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:50.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:50 smithi033 ceph-mon[97738]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:50.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:50 smithi033 ceph-mon[101783]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:50.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:15:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:15:50] "GET /metrics HTTP/1.1" 200 34220 "" "Prometheus/2.43.0" 2024-04-05T03:15:51.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:50 smithi151 ceph-mon[88672]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:52.416 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:52.692 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:52.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:52 smithi033 ceph-mon[97738]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:52.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:52 smithi033 ceph-mon[101783]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:52.829 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:52 smithi151 ceph-mon[88672]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:54.497 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:15:54.497 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:54.879 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:54 smithi033 ceph-mon[97738]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:54.879 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:54 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2755269056' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:54.879 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:54 smithi033 ceph-mon[101783]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:54.879 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:54 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2755269056' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:55.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:54 smithi151 ceph-mon[88672]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:55.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:54 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2755269056' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:56 smithi033 ceph-mon[97738]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:56.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:56 smithi033 ceph-mon[101783]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:57.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:56 smithi151 ceph-mon[88672]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:58.092 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:58.367 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:58.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:58 smithi033 ceph-mon[97738]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:58.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:58 smithi033 ceph-mon[101783]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:59.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:58 smithi151 ceph-mon[88672]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:59.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:15:59 smithi033 ceph-mon[97738]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:15:59.712 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:15:59 smithi033 ceph-mon[101783]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:00.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:15:59 smithi151 ceph-mon[88672]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:00.217 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:00.217 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:00.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:00 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/975787100' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:00.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:00 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/975787100' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:01.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:00 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/975787100' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:01.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:16:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:16:00] "GET /metrics HTTP/1.1" 200 34220 "" "Prometheus/2.43.0" 2024-04-05T03:16:01.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:01 smithi033 ceph-mon[97738]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:01.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:01 smithi033 ceph-mon[101783]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:01 smithi151 ceph-mon[88672]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:02.979 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:02.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:03.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:03.824 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:03.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:03 smithi033 ceph-mon[97738]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:03.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:03 smithi033 ceph-mon[101783]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:04.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:03 smithi151 ceph-mon[88672]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:04.097 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:05.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:05 smithi033 ceph-mon[97738]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:05.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:05 smithi033 ceph-mon[101783]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:05.993 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:05.993 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:06.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:05 smithi151 ceph-mon[88672]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:06.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:06 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1725545963' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:06.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:06 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1725545963' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:07.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:06 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1725545963' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:07.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:07 smithi033 ceph-mon[97738]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:07.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:07 smithi033 ceph-mon[101783]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:08.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:07 smithi151 ceph-mon[88672]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:09.679 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:09.954 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:09.979 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:09 smithi033 ceph-mon[101783]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:09.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:09 smithi033 ceph-mon[97738]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:10.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:09 smithi151 ceph-mon[88672]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:11.119 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:16:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:16:10] "GET /metrics HTTP/1.1" 200 34220 "" "Prometheus/2.43.0" 2024-04-05T03:16:11.948 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:11.948 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:12.540 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:12 smithi033 ceph-mon[97738]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:12.540 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:12 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1430510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:12.540 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:12 smithi033 ceph-mon[101783]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:12.540 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:12 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1430510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:12.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:12 smithi151 ceph-mon[88672]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:12.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:12 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1430510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:14.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:14 smithi033 ceph-mon[101783]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:14.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:14 smithi033 ceph-mon[97738]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:14.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:14 smithi151 ceph-mon[88672]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:15.543 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:15.816 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:16.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:16 smithi033 ceph-mon[97738]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:16 smithi033 ceph-mon[101783]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:16.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:16 smithi151 ceph-mon[88672]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:17.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:17.879 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:17.879 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:17.879 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:17.880 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:18.578 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:18 smithi033 ceph-mon[101783]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:18.579 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:18 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1403186068' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:18.579 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:18 smithi033 ceph-mon[97738]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:18.579 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:18 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1403186068' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:18.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:18 smithi151 ceph-mon[88672]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:18.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:18 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1403186068' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:20.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:20 smithi033 ceph-mon[97738]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:20.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:20 smithi033 ceph-mon[101783]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:20.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:20 smithi151 ceph-mon[88672]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:21.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:16:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:16:20] "GET /metrics HTTP/1.1" 200 34220 "" "Prometheus/2.43.0" 2024-04-05T03:16:21.606 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:21.880 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:22.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:22 smithi033 ceph-mon[97738]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:22 smithi033 ceph-mon[101783]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:22 smithi151 ceph-mon[88672]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:23.940 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:23.941 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:24.620 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:24 smithi033 ceph-mon[97738]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:24.620 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:24 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/437130345' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:24.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:24 smithi033 ceph-mon[101783]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:24.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:24 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/437130345' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:24.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:24 smithi151 ceph-mon[88672]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:24.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:24 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/437130345' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:25.756 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:25 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:16:25.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:25 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:16:25.758 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:25 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:16:26.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:26 smithi151 ceph-mon[88672]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:26.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:26 smithi033 ceph-mon[97738]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:26.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:26 smithi033 ceph-mon[101783]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:27.650 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:28.567 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:28.691 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:28 smithi033 ceph-mon[97738]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:28.691 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:28 smithi033 ceph-mon[101783]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:28.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:28 smithi151 ceph-mon[88672]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:29.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:16:29.781 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:30.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:30.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:30.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:30.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:30.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:16:30.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:16:30.028 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:16:30.423 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:30.423 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:30.712 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[101783]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:30.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[101783]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:30.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[101783]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:30.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/4168261836' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:30.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[97738]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:30.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[97738]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:30.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[97738]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:30.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/4168261836' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:30.980 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:16:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:16:30] "GET /metrics HTTP/1.1" 200 34220 "" "Prometheus/2.43.0" 2024-04-05T03:16:31.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:30 smithi151 ceph-mon[88672]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:31.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:30 smithi151 ceph-mon[88672]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:31.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:30 smithi151 ceph-mon[88672]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:31.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:30 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/4168261836' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:32.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:32 smithi033 ceph-mon[97738]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:32.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:32.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:32 smithi033 ceph-mon[101783]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:32.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:33.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:32 smithi151 ceph-mon[88672]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:33.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:34.188 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:34.465 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:34.867 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:34 smithi033 ceph-mon[97738]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:34.867 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:34 smithi033 ceph-mon[101783]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:35.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:34 smithi151 ceph-mon[88672]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:36.547 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:36.548 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:36.945 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:36 smithi033 ceph-mon[101783]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:36.945 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:36 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1089724391' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:36.945 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:36 smithi033 ceph-mon[97738]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:36.946 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:36 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1089724391' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:37.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:36 smithi151 ceph-mon[88672]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:37.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:36 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1089724391' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:37.979 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:37 smithi033 ceph-mon[101783]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:37.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:37 smithi033 ceph-mon[97738]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:38.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:37 smithi151 ceph-mon[88672]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:40.367 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:40.641 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:40.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:40 smithi033 ceph-mon[97738]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:40.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:40 smithi033 ceph-mon[101783]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:40.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:40 smithi151 ceph-mon[88672]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:41.002 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:16:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:16:40] "GET /metrics HTTP/1.1" 200 34216 "" "Prometheus/2.43.0" 2024-04-05T03:16:42.474 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:42.474 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:42.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:42 smithi033 ceph-mon[97738]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:42.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:42 smithi033 ceph-mon[101783]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:42.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:42 smithi151 ceph-mon[88672]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:43.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:43 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2040749293' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:43.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:43 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2040749293' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:43.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:43 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2040749293' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:44.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:44 smithi033 ceph-mon[97738]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:44.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:44 smithi033 ceph-mon[101783]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:44.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:44 smithi151 ceph-mon[88672]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:46.075 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:46.352 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:46.631 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:46 smithi033 ceph-mon[97738]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:46.632 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:46 smithi033 ceph-mon[101783]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:46.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:46 smithi151 ceph-mon[88672]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:47.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:47.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:16:48.377 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:48.377 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:48.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:48 smithi033 ceph-mon[97738]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:48.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:48 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1267710872' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:48.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:48 smithi033 ceph-mon[101783]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:48.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:48 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1267710872' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:48.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:48 smithi151 ceph-mon[88672]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:48.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:48 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1267710872' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:50.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:50 smithi033 ceph-mon[97738]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:50.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:50 smithi033 ceph-mon[101783]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:50.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:50 smithi151 ceph-mon[88672]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:51.229 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:16:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:16:50] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2024-04-05T03:16:52.130 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:52.404 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:52.479 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:52 smithi033 ceph-mon[97738]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:52.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:52 smithi033 ceph-mon[101783]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:52.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:52 smithi151 ceph-mon[88672]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:54.264 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:16:54.264 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:16:54.624 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:54 smithi033 ceph-mon[101783]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:54.624 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:54 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2502906006' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:54.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:54 smithi033 ceph-mon[97738]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:54.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:54 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2502906006' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:54.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:54 smithi151 ceph-mon[88672]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:54.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:54 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2502906006' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:16:56.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:56 smithi033 ceph-mon[97738]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:56.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:56 smithi033 ceph-mon[101783]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:56.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:56 smithi151 ceph-mon[88672]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:57.952 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:16:58.225 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:16:58.686 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:16:58 smithi033 ceph-mon[97738]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:58.686 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:16:58 smithi033 ceph-mon[101783]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:16:58.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:16:58 smithi151 ceph-mon[88672]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:00.094 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:00.094 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:00.672 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:00 smithi033 ceph-mon[97738]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:00.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:00 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3767427539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:00.673 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:00 smithi033 ceph-mon[101783]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:00.673 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:00 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3767427539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:00.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:00 smithi151 ceph-mon[88672]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:00.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:00 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3767427539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:00.979 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:17:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:17:00] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2024-04-05T03:17:02.715 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:02 smithi033 ceph-mon[101783]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:02.716 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:02.716 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:02 smithi033 ceph-mon[97738]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:02.716 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:02.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:02 smithi151 ceph-mon[88672]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:02.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:03.699 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:03.965 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:04.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:04 smithi033 ceph-mon[97738]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:04.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:04 smithi033 ceph-mon[101783]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:04.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:04 smithi151 ceph-mon[88672]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:05.974 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:05.975 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:06.662 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:06 smithi033 ceph-mon[101783]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:06.663 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:06 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/217982489' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:06.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:06 smithi033 ceph-mon[97738]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:06.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:06 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/217982489' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:06.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:06 smithi151 ceph-mon[88672]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:06.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:06 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/217982489' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:08.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:08 smithi033 ceph-mon[97738]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:08.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:08 smithi033 ceph-mon[101783]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:08.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:08 smithi151 ceph-mon[88672]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:09.690 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:09.962 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:10.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:10 smithi033 ceph-mon[101783]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:10.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:10 smithi033 ceph-mon[97738]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:10.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:10 smithi151 ceph-mon[88672]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:11.087 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:17:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:17:10] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2024-04-05T03:17:11.832 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:11.832 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:12.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:12 smithi033 ceph-mon[97738]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:12.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:12 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3677034187' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:12.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:12 smithi033 ceph-mon[101783]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:12.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:12 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3677034187' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:12 smithi151 ceph-mon[88672]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:12.778 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:12 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3677034187' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:14.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:14 smithi033 ceph-mon[97738]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:14.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:14 smithi033 ceph-mon[101783]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:14.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:14 smithi151 ceph-mon[88672]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:15.529 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:15.801 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:16.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:16 smithi033 ceph-mon[97738]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:16.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:16 smithi033 ceph-mon[101783]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:16.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:16 smithi151 ceph-mon[88672]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:17.717 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:17.717 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:17.718 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:17.718 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:17.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:18 smithi033 ceph-mon[97738]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:18.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:18 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2816524495' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:18.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:18 smithi033 ceph-mon[101783]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:18.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:18 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2816524495' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:18.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:18 smithi151 ceph-mon[88672]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:18.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:18 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2816524495' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:20.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:20 smithi033 ceph-mon[97738]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:20.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:20 smithi033 ceph-mon[101783]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:20.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:20 smithi151 ceph-mon[88672]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:21.229 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:17:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:17:20] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2024-04-05T03:17:21.433 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:21.710 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:22.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:22 smithi033 ceph-mon[97738]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:22.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:22 smithi033 ceph-mon[101783]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:22.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:22 smithi151 ceph-mon[88672]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:23.712 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:23.713 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:24 smithi033 ceph-mon[97738]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:24.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:24 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3297250465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:24.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:24 smithi033 ceph-mon[101783]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:24.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:24 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3297250465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:24.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:24 smithi151 ceph-mon[88672]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:24.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:24 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3297250465' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:26.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:26 smithi033 ceph-mon[97738]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:26.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:26 smithi033 ceph-mon[101783]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:26.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:26 smithi151 ceph-mon[88672]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:27.266 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:27.541 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:28.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:28 smithi033 ceph-mon[97738]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:28.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:28 smithi033 ceph-mon[101783]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:28.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:28 smithi151 ceph-mon[88672]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:29.380 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:29.380 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:29.723 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:29 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3592421363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:29.723 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:29 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:17:29.723 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:29 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3592421363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:29.723 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:29 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:17:29.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:29 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3592421363' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:29.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:29 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:17:30.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[101783]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[97738]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.735 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:30 smithi151 ceph-mon[88672]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:30.735 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.735 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.735 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:30.735 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:30 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:31.028 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:17:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:17:30] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2024-04-05T03:17:32.541 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:32 smithi151 ceph-mon[88672]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:32.541 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:32.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:32 smithi033 ceph-mon[97738]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:32.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:32.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:32 smithi033 ceph-mon[101783]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:32.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:33.060 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:33.700 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:34.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[97738]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:34.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[101783]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:34 smithi151 ceph-mon[88672]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:34.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:34.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:35.527 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:35.527 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:35.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:17:35.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[101783]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[101783]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[97738]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[97738]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:35.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:35 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:35.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:35 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:17:35.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:35 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:17:35.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:35 smithi151 ceph-mon[88672]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:35.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:35 smithi151 ceph-mon[88672]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:35.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:35 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:17:36.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:36 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/514037483' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:36.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:36 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/514037483' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:36.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:36 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/514037483' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:37.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:37 smithi151 ceph-mon[88672]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:37.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:37 smithi033 ceph-mon[97738]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:37.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:37 smithi033 ceph-mon[101783]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:39.078 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:39.354 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:39.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:39 smithi033 ceph-mon[97738]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:39.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:39 smithi033 ceph-mon[101783]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:39.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:39 smithi151 ceph-mon[88672]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:41.230 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:17:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:17:40] "GET /metrics HTTP/1.1" 200 34222 "" "Prometheus/2.43.0" 2024-04-05T03:17:41.235 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:41.235 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:41.800 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:41 smithi033 ceph-mon[101783]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:41.801 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:41 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/4037309688' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:41.801 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:41 smithi033 ceph-mon[97738]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:41.801 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:41 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/4037309688' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:42.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:41 smithi151 ceph-mon[88672]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:42.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:41 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/4037309688' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:43.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:43 smithi033 ceph-mon[97738]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:43.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:43 smithi033 ceph-mon[101783]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:44.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:43 smithi151 ceph-mon[88672]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:44.829 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:45.104 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:45.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:45 smithi033 ceph-mon[97738]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:45.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:45 smithi033 ceph-mon[101783]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:46.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:45 smithi151 ceph-mon[88672]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:47.157 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:47.157 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:47.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:47 smithi033 ceph-mon[101783]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:47.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:47 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1254260243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:47 smithi033 ceph-mon[97738]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:47 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1254260243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:48.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:47 smithi151 ceph-mon[88672]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:48.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:17:48.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:47 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1254260243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:49.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:49 smithi033 ceph-mon[97738]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:49.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:49 smithi033 ceph-mon[101783]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:50.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:49 smithi151 ceph-mon[88672]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:50.760 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:51.034 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:51.157 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:17:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:17:50] "GET /metrics HTTP/1.1" 200 34218 "" "Prometheus/2.43.0" 2024-04-05T03:17:51.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:51 smithi033 ceph-mon[97738]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:51.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:51 smithi033 ceph-mon[101783]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:52.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:51 smithi151 ceph-mon[88672]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:52.919 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:52.919 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:53.829 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:53 smithi151 ceph-mon[88672]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:53.829 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:53 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1060206985' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:53.979 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:53 smithi033 ceph-mon[101783]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:53.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:53 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1060206985' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:53.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:53 smithi033 ceph-mon[97738]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:53.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:53 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1060206985' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:55.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:55 smithi033 ceph-mon[97738]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:55.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:55 smithi033 ceph-mon[101783]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:56.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:55 smithi151 ceph-mon[88672]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:56.573 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:17:56.849 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:17:57.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:57 smithi033 ceph-mon[97738]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:57.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:57 smithi033 ceph-mon[101783]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:58.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:57 smithi151 ceph-mon[88672]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:58.706 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:17:58.706 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:17:59.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:59 smithi033 ceph-mon[101783]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:59.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:17:59 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/181059299' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:17:59.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:59 smithi033 ceph-mon[97738]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:17:59.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:17:59 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/181059299' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:00.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:59 smithi151 ceph-mon[88672]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:00.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:17:59 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/181059299' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:01.229 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:18:00] "GET /metrics HTTP/1.1" 200 34218 "" "Prometheus/2.43.0" 2024-04-05T03:18:01.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:01 smithi033 ceph-mon[97738]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:01.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:01 smithi033 ceph-mon[101783]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:01 smithi151 ceph-mon[88672]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:02.440 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:02.705 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:02.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:02 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:02.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:02 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:03.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:02 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:03.812 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:03 smithi033 ceph-mon[97738]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:03.813 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:03 smithi033 ceph-mon[101783]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:04.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:03 smithi151 ceph-mon[88672]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:04.566 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:04.566 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:05.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:05 smithi033 ceph-mon[97738]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:05.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:05 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2312870680' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:05.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:05 smithi033 ceph-mon[101783]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:05.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:05 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2312870680' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:06.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:05 smithi151 ceph-mon[88672]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:06.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:05 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2312870680' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:07.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:07 smithi033 ceph-mon[97738]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:07.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:07 smithi033 ceph-mon[101783]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:08.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:07 smithi151 ceph-mon[88672]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:08.266 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:08.539 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:08.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:08 smithi033 ceph-mon[101783]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:08.924 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:08 smithi033 ceph-mon[97738]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:09.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:08 smithi151 ceph-mon[88672]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:10.524 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:10.524 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:10.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:10 smithi033 ceph-mon[97738]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:10.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:10 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/214532041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:10.877 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:10 smithi033 ceph-mon[101783]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:10.877 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:10 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/214532041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:10.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:10 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:18:10] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2024-04-05T03:18:11.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:10 smithi151 ceph-mon[88672]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:11.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:10 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/214532041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:13.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:13 smithi033 ceph-mon[97738]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:13.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:13 smithi033 ceph-mon[101783]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:14.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:13 smithi151 ceph-mon[88672]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:14.298 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:14.571 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:15.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:15 smithi033 ceph-mon[97738]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:15.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:15 smithi033 ceph-mon[101783]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:16.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:15 smithi151 ceph-mon[88672]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:16.483 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:16.483 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:16.930 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:16 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1137811014' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:16.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:16 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1137811014' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:17.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:16 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1137811014' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:17.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:17 smithi033 ceph-mon[97738]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:17.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:17 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:17.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:17 smithi033 ceph-mon[101783]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:17.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:17 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:18.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:17 smithi151 ceph-mon[88672]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:18.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:17 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:19.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:19 smithi033 ceph-mon[97738]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:19.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:19 smithi033 ceph-mon[101783]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:20.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:19 smithi151 ceph-mon[88672]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:20.167 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:20.440 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:20.888 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:20 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:18:20] "GET /metrics HTTP/1.1" 200 34218 "" "Prometheus/2.43.0" 2024-04-05T03:18:21.843 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:21 smithi033 ceph-mon[101783]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:21.843 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:21 smithi033 ceph-mon[97738]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:22.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:21 smithi151 ceph-mon[88672]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:22.324 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:22.324 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:22.648 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:22 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/269537967' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:22.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:22 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/269537967' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:23.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:22 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/269537967' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:23.930 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:23 smithi151 ceph-mon[88672]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:23.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:23 smithi033 ceph-mon[97738]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:23.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:23 smithi033 ceph-mon[101783]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:25.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:25 smithi033 ceph-mon[97738]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:25.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:25 smithi033 ceph-mon[101783]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:25.985 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:26.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:25 smithi151 ceph-mon[88672]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:26.258 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:27.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:27 smithi033 ceph-mon[97738]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:27.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:27 smithi033 ceph-mon[101783]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:28.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:27 smithi151 ceph-mon[88672]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:28.269 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:28.269 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:28.880 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:28 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2502294223' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:28.881 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:28 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2502294223' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:29.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:28 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2502294223' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:29.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:29 smithi033 ceph-mon[97738]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:29.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:29 smithi033 ceph-mon[101783]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:30.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:29 smithi151 ceph-mon[88672]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:31.229 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:30 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:18:30] "GET /metrics HTTP/1.1" 200 34218 "" "Prometheus/2.43.0" 2024-04-05T03:18:31.907 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:31.937 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:31 smithi033 ceph-mon[97738]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:31.937 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:31 smithi033 ceph-mon[101783]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:32.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:31 smithi151 ceph-mon[88672]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:32.186 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:32.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:32 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:32.980 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:32 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:33.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:32 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:33.959 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:33 smithi033 ceph-mon[97738]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:33.959 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:33 smithi033 ceph-mon[101783]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:34.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:33 smithi151 ceph-mon[88672]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:34.167 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:34.167 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:34.880 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:34 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/1937472180' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:34.880 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:34 smithi033 ceph-mon[97738]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:34.881 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:34 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:18:34.881 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:34 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/1937472180' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:34.881 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:34 smithi033 ceph-mon[101783]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:34.881 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:34 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:18:35.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:34 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/1937472180' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:35.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:34 smithi151 ceph-mon[88672]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:35.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:34 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:18:37.819 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:37 smithi033 ceph-mon[101783]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:37.819 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:37 smithi033 ceph-mon[97738]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:37.895 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:37 smithi151 ceph-mon[88672]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:37.906 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:38.173 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:39.378 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[101783]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.379 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:39 smithi033 ceph-mon[97738]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:39.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:39 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:39.527 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:39 smithi151 ceph-mon[88672]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:40.186 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:40.187 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:40.479 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:18:40.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:18:40.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:40.480 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/3818985874' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:40.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:18:40.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:18:40.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:40.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/3818985874' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:40.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:40 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:18:40.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:40 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:18:40.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:40 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' 2024-04-05T03:18:40.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:40 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/3818985874' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:40.958 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:40 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:18:40] "GET /metrics HTTP/1.1" 200 34220 "" "Prometheus/2.43.0" 2024-04-05T03:18:41.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:41 smithi033 ceph-mon[97738]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:41.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:41 smithi033 ceph-mon[97738]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:41 smithi033 ceph-mon[101783]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:41.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:41 smithi033 ceph-mon[101783]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:41.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:41 smithi151 ceph-mon[88672]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:41.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:41 smithi151 ceph-mon[88672]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:43.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:43 smithi033 ceph-mon[97738]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:43.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:43 smithi033 ceph-mon[101783]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:43.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:43 smithi151 ceph-mon[88672]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:43.986 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:44.264 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:45.598 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:45 smithi033 ceph-mon[97738]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:45.598 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:45 smithi033 ceph-mon[101783]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:45.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:45 smithi151 ceph-mon[88672]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:46.193 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:46.193 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:46.479 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:46 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2070411071' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:46.480 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:46 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2070411071' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:46.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:46 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2070411071' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:47.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:47 smithi033 ceph-mon[101783]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:47.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:47 smithi033 ceph-mon[101783]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:47 smithi033 ceph-mon[97738]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:47.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:47 smithi033 ceph-mon[97738]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:47.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:47 smithi151 ceph-mon[88672]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:47.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:47 smithi151 ceph-mon[88672]: from='mgr.25934 172.21.15.33:0/86340533' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:18:49.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:49 smithi033 ceph-mon[97738]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:49.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:49 smithi033 ceph-mon[101783]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:49.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:49 smithi151 ceph-mon[88672]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:49.940 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:50.217 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:50.979 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:50 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:18:50] "GET /metrics HTTP/1.1" 200 34215 "" "Prometheus/2.43.0" 2024-04-05T03:18:51.590 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:51 smithi033 ceph-mon[101783]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:51.590 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:51 smithi033 ceph-mon[97738]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:51.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:51 smithi151 ceph-mon[88672]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:52.056 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:52.056 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:52.704 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:52 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/4102797599' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:52.704 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:52 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/4102797599' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:52.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:52 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/4102797599' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:53.678 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:53 smithi151 ceph-mon[88672]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:53.729 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:53 smithi033 ceph-mon[97738]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:53.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:53 smithi033 ceph-mon[101783]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:55.730 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:55 smithi033 ceph-mon[97738]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:55.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:55 smithi033 ceph-mon[101783]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:55.731 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:18:55.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:55 smithi151 ceph-mon[88672]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:56.004 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:18:57.598 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:57 smithi033 ceph-mon[97738]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:57.599 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:57 smithi033 ceph-mon[101783]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:57.777 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:57 smithi151 ceph-mon[88672]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-04-05T03:18:57.857 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-04-05T03:18:57.857 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:18:58.569 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:18:58 smithi033 ceph-mon[101783]: from='client.? 172.21.15.33:0/2087384356' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:58.569 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:58 smithi033 ceph-mon[97738]: from='client.? 172.21.15.33:0/2087384356' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:58.600 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-05T03:18:58.832 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=4a685f6e14724085a068ed25a2383fb4 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-05T03:18:58.835 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-05T03:18:58.845 INFO:tasks.cephadm:Teardown begin 2024-04-05T03:18:58.845 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:18:58.849 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:18:58 smithi151 ceph-mon[88672]: from='client.? 172.21.15.33:0/2087384356' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:18:58.883 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:18:58.918 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-05T03:18:58.918 DEBUG:teuthology.orchestra.run.smithi033:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-05T03:18:58.939 DEBUG:teuthology.orchestra.run.smithi151:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-05T03:18:58.976 INFO:tasks.cephadm:Stopping all daemons... 2024-04-05T03:18:58.976 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-05T03:18:58.977 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a 2024-04-05T03:18:59.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:59 smithi033 systemd[1]: Stopping Ceph mon.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:18:59.657 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a[97734]: 2024-04-05T03:18:59.291+0000 7f2901a93700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:18:59.657 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a[97734]: 2024-04-05T03:18:59.291+0000 7f2901a93700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-05T03:18:59.657 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:59 smithi033 podman[143699]: 2024-04-05 03:18:59.372994088 +0000 UTC m=+0.189993023 container died 7657e117c64ff20299b2edb0722221d7da3578304e1b51824221e624a415d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-0028525, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-05T03:18:59.657 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:18:59] ENGINE Bus STOPPING 2024-04-05T03:18:59.658 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:18:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:18:59.658 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:18:59] ENGINE Bus STOPPED 2024-04-05T03:18:59.658 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:18:59] ENGINE Bus STARTING 2024-04-05T03:18:59.658 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:18:59] ENGINE Serving on http://:::9283 2024-04-05T03:18:59.658 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:18:59 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:18:59] ENGINE Bus STARTED 2024-04-05T03:18:59.979 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:59 smithi033 podman[143699]: 2024-04-05 03:18:59.758301035 +0000 UTC m=+0.575299968 container cleanup 7657e117c64ff20299b2edb0722221d7da3578304e1b51824221e624a415d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:18:59.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Apr 05 03:18:59 smithi033 bash[143699]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-a 2024-04-05T03:19:00.117 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.a.service' 2024-04-05T03:19:00.169 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:00.169 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-05T03:19:00.169 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-05T03:19:00.170 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.c 2024-04-05T03:19:00.469 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:19:00 smithi033 systemd[1]: Stopping Ceph mon.c for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:00.729 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c[101776]: 2024-04-05T03:19:00.470+0000 7f71874c6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:00.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c[101776]: 2024-04-05T03:19:00.470+0000 7f71874c6700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-05T03:19:00.730 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:19:00 smithi033 podman[143833]: 2024-04-05 03:19:00.531836808 +0000 UTC m=+0.175530948 container died fb77798ae736f6b2fd218b802ece033ee10c821766fb2f3fc5b6f901e092f69c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD) 2024-04-05T03:19:00.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:19:00] ENGINE Bus STOPPING 2024-04-05T03:19:00.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:19:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:19:00.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:19:00] ENGINE Bus STOPPED 2024-04-05T03:19:00.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:19:00] ENGINE Bus STARTING 2024-04-05T03:19:00.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:19:00] ENGINE Serving on http://:::9283 2024-04-05T03:19:00.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: [05/Apr/2024:03:19:00] ENGINE Bus STARTED 2024-04-05T03:19:01.162 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:00 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: ::ffff:172.21.15.151 - - [05/Apr/2024:03:19:00] "GET /metrics HTTP/1.1" 200 34215 "" "Prometheus/2.43.0" 2024-04-05T03:19:01.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:19:00 smithi033 podman[143833]: 2024-04-05 03:19:00.887363743 +0000 UTC m=+0.531057871 container cleanup fb77798ae736f6b2fd218b802ece033ee10c821766fb2f3fc5b6f901e092f69c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True) 2024-04-05T03:19:01.162 INFO:journalctl@ceph.mon.c.smithi033.stdout:Apr 05 03:19:00 smithi033 bash[143833]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-c 2024-04-05T03:19:01.320 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.c.service' 2024-04-05T03:19:01.358 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:01.358 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-05T03:19:01.358 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-05T03:19:01.358 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.b 2024-04-05T03:19:01.690 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:01 smithi151 systemd[1]: Stopping Ceph mon.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b[88668]: 2024-04-05T03:19:01.689+0000 7fdc1312a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b[88668]: 2024-04-05T03:19:01.690+0000 7fdc1312a700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-05T03:19:02.027 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:01 smithi151 podman[98809]: 2024-04-05 03:19:01.864778212 +0000 UTC m=+0.306132909 container died 634be946ebb9a25aefab251da1f74329c86b43354fed1500f82fe92f4ead4737 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, ceph=True) 2024-04-05T03:19:02.482 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:02 smithi151 podman[98809]: 2024-04-05 03:19:02.307489859 +0000 UTC m=+0.748844546 container cleanup 634be946ebb9a25aefab251da1f74329c86b43354fed1500f82fe92f4ead4737 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:19:02.482 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:02 smithi151 bash[98809]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b 2024-04-05T03:19:02.722 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.b.service' 2024-04-05T03:19:02.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:02 smithi151 podman[98823]: 2024-04-05 03:19:02.48406358 +0000 UTC m=+0.616423778 container remove 634be946ebb9a25aefab251da1f74329c86b43354fed1500f82fe92f4ead4737 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mon-b, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-05T03:19:02.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:02 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.b.service: Deactivated successfully. 2024-04-05T03:19:02.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:02 smithi151 systemd[1]: Stopped Ceph mon.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:19:02.746 INFO:journalctl@ceph.mon.b.smithi151.stdout:Apr 05 03:19:02 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mon.b.service: Consumed 5.817s CPU time. 2024-04-05T03:19:02.788 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:02.788 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-05T03:19:02.788 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-05T03:19:02.788 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y 2024-04-05T03:19:02.979 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:02 smithi033 systemd[1]: Stopping Ceph mgr.y for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:03.466 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:03 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:19:03.117+0000 7f51a8777700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:03.467 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:03 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y[88566]: 2024-04-05T03:19:03.117+0000 7f51a8777700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-05T03:19:03.467 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:03 smithi033 podman[143975]: 2024-04-05 03:19:03.148383921 +0000 UTC m=+0.137365523 container died a6e11bc7dc6c105c336170436d37ec1f1e4d4ab7ac7154080371b6bc1da59077 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, RELEASE=squid-0028525, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240403, io.buildah.version=1.33.5) 2024-04-05T03:19:03.729 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:03 smithi033 podman[143975]: 2024-04-05 03:19:03.61729821 +0000 UTC m=+0.606279807 container cleanup a6e11bc7dc6c105c336170436d37ec1f1e4d4ab7ac7154080371b6bc1da59077 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525) 2024-04-05T03:19:03.730 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Apr 05 03:19:03 smithi033 bash[143975]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-y 2024-04-05T03:19:04.033 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.y.service' 2024-04-05T03:19:04.072 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:04.072 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-05T03:19:04.072 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-05T03:19:04.072 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x 2024-04-05T03:19:04.425 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:19:04 smithi151 systemd[1]: Stopping Ceph mgr.x for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:04.703 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:19:04 smithi151 podman[98931]: 2024-04-05 03:19:04.426983927 +0000 UTC m=+0.160354028 container died 090d83bbd680c504ba3b7e5bc6d6e695e1a3443f2496965cd3363054e6ee3871 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-05T03:19:05.027 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:19:04 smithi151 podman[98931]: 2024-04-05 03:19:04.787656629 +0000 UTC m=+0.521026746 container cleanup 090d83bbd680c504ba3b7e5bc6d6e695e1a3443f2496965cd3363054e6ee3871 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-05T03:19:05.028 INFO:journalctl@ceph.mgr.x.smithi151.stdout:Apr 05 03:19:04 smithi151 bash[98931]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-mgr-x 2024-04-05T03:19:05.161 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@mgr.x.service' 2024-04-05T03:19:05.198 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:05.199 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-05T03:19:05.199 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-05T03:19:05.200 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.0 2024-04-05T03:19:05.504 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:05 smithi033 systemd[1]: Stopping Ceph osd.0 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:05.979 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:05 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[109236]: 2024-04-05T03:19:05.504+0000 7fb3f1369700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:05.980 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:05 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[109236]: 2024-04-05T03:19:05.504+0000 7fb3f1369700 -1 osd.0 108 *** Got signal Terminated *** 2024-04-05T03:19:05.980 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:05 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0[109236]: 2024-04-05T03:19:05.504+0000 7fb3f1369700 -1 osd.0 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:19:11.087 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:10 smithi033 podman[144100]: 2024-04-05 03:19:10.810890629 +0000 UTC m=+5.408350445 container died 6a19fd0140d9f1f4cc63d4c79cf41b2f07a76c9038dcb9a6f48c4d2db2f6436c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-0028525) 2024-04-05T03:19:11.446 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:11 smithi033 podman[144100]: 2024-04-05 03:19:11.255089954 +0000 UTC m=+5.852549760 container cleanup 6a19fd0140d9f1f4cc63d4c79cf41b2f07a76c9038dcb9a6f48c4d2db2f6436c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-05T03:19:11.447 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:11 smithi033 bash[144100]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0 2024-04-05T03:19:11.729 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:11 smithi033 podman[144117]: 2024-04-05 03:19:11.448638108 +0000 UTC m=+0.627405154 container remove 6a19fd0140d9f1f4cc63d4c79cf41b2f07a76c9038dcb9a6f48c4d2db2f6436c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-05T03:19:12.200 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:11 smithi033 podman[144205]: 2024-04-05 03:19:11.780553666 +0000 UTC m=+0.018291481 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:19:12.200 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:11 smithi033 podman[144205]: 2024-04-05 03:19:11.882881525 +0000 UTC m=+0.120619370 container create 42dff470f686b3f594e7809213d3c0857eda55a10cb0237ce381ce95db9e3595 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, org.label-schema.build-date=20240403, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:19:12.200 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:12 smithi033 podman[144205]: 2024-04-05 03:19:12.085126899 +0000 UTC m=+0.322864698 container init 42dff470f686b3f594e7809213d3c0857eda55a10cb0237ce381ce95db9e3595 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, RELEASE=squid-0028525, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240403, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-05T03:19:12.200 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:12 smithi033 podman[144205]: 2024-04-05 03:19:12.088056335 +0000 UTC m=+0.325794143 container start 42dff470f686b3f594e7809213d3c0857eda55a10cb0237ce381ce95db9e3595 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-04-05T03:19:12.200 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:12 smithi033 podman[144205]: 2024-04-05 03:19:12.201745135 +0000 UTC m=+0.439482935 container attach 42dff470f686b3f594e7809213d3c0857eda55a10cb0237ce381ce95db9e3595 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-05T03:19:12.479 INFO:journalctl@ceph.osd.0.smithi033.stdout:Apr 05 03:19:12 smithi033 podman[144205]: 2024-04-05 03:19:12.380505351 +0000 UTC m=+0.618243151 container died 42dff470f686b3f594e7809213d3c0857eda55a10cb0237ce381ce95db9e3595 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-0-deactivate, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, ceph=True) 2024-04-05T03:19:12.993 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.0.service' 2024-04-05T03:19:13.035 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:13.035 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-05T03:19:13.035 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-05T03:19:13.035 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.1 2024-04-05T03:19:13.229 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:13 smithi033 systemd[1]: Stopping Ceph osd.1 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:13.729 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[114474]: 2024-04-05T03:19:13.385+0000 7f1b84e5a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:13.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[114474]: 2024-04-05T03:19:13.385+0000 7f1b84e5a700 -1 osd.1 108 *** Got signal Terminated *** 2024-04-05T03:19:13.730 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:13 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1[114474]: 2024-04-05T03:19:13.385+0000 7f1b84e5a700 -1 osd.1 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:19:14.729 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:14.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:19:14.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:14 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:14.277Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:19:19.108 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:18 smithi033 podman[144311]: 2024-04-05 03:19:18.788183048 +0000 UTC m=+5.519667449 container died 22dcd72df86aae5032f9c56a2b1ebdde14628453523601ac09eaa722ff0a7460 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-04-05T03:19:19.480 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:19 smithi033 podman[144311]: 2024-04-05 03:19:19.259950598 +0000 UTC m=+5.991435031 container cleanup 22dcd72df86aae5032f9c56a2b1ebdde14628453523601ac09eaa722ff0a7460 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525) 2024-04-05T03:19:19.480 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:19 smithi033 bash[144311]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1 2024-04-05T03:19:19.837 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:19 smithi033 podman[144329]: 2024-04-05 03:19:19.519595942 +0000 UTC m=+0.727764688 container remove 22dcd72df86aae5032f9c56a2b1ebdde14628453523601ac09eaa722ff0a7460 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-05T03:19:20.154 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:19 smithi033 podman[144419]: 2024-04-05 03:19:19.742374934 +0000 UTC m=+0.018090779 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:19:20.154 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:19 smithi033 podman[144419]: 2024-04-05 03:19:19.860893688 +0000 UTC m=+0.136609504 container create ccdc1382352629bc336bce8f35b49d1cefe3f9a1f621d38420dcb2247dede4a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-05T03:19:20.154 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:20 smithi033 podman[144419]: 2024-04-05 03:19:20.064752055 +0000 UTC m=+0.340467873 container init ccdc1382352629bc336bce8f35b49d1cefe3f9a1f621d38420dcb2247dede4a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-05T03:19:20.155 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:20 smithi033 podman[144419]: 2024-04-05 03:19:20.067730053 +0000 UTC m=+0.343445875 container start ccdc1382352629bc336bce8f35b49d1cefe3f9a1f621d38420dcb2247dede4a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-05T03:19:20.155 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:20 smithi033 podman[144419]: 2024-04-05 03:19:20.15629756 +0000 UTC m=+0.432013395 container attach ccdc1382352629bc336bce8f35b49d1cefe3f9a1f621d38420dcb2247dede4a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240403) 2024-04-05T03:19:20.479 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:20 smithi033 podman[144419]: 2024-04-05 03:19:20.362469753 +0000 UTC m=+0.638185581 container died ccdc1382352629bc336bce8f35b49d1cefe3f9a1f621d38420dcb2247dede4a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-05T03:19:20.950 INFO:journalctl@ceph.osd.1.smithi033.stdout:Apr 05 03:19:20 smithi033 podman[144445]: 2024-04-05 03:19:20.934985759 +0000 UTC m=+0.566227108 container remove ccdc1382352629bc336bce8f35b49d1cefe3f9a1f621d38420dcb2247dede4a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-0028525, CEPH_POINT_RELEASE=) 2024-04-05T03:19:20.956 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.1.service' 2024-04-05T03:19:20.999 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:20.999 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-05T03:19:20.999 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-05T03:19:20.999 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.2 2024-04-05T03:19:21.229 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:21 smithi033 systemd[1]: Stopping Ceph osd.2 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:21.729 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:21 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[119358]: 2024-04-05T03:19:21.370+0000 7ffbc272a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:21.729 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:21 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[119358]: 2024-04-05T03:19:21.370+0000 7ffbc272a700 -1 osd.2 108 *** Got signal Terminated *** 2024-04-05T03:19:21.730 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:21 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2[119358]: 2024-04-05T03:19:21.370+0000 7ffbc272a700 -1 osd.2 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:19:24.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:24.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:19:24.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:24.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:19:24.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:24 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:24.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:19:26.945 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:26 smithi033 podman[144525]: 2024-04-05 03:19:26.68048632 +0000 UTC m=+5.407694654 container died cfa7f55e5875d8d4118b71ad51d32aa910445ea51c4b9649521174c51eedaf52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-05T03:19:27.229 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:27 smithi033 podman[144525]: 2024-04-05 03:19:27.112723131 +0000 UTC m=+5.839931499 container cleanup cfa7f55e5875d8d4118b71ad51d32aa910445ea51c4b9649521174c51eedaf52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:19:27.230 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:27 smithi033 bash[144525]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2 2024-04-05T03:19:27.729 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:27 smithi033 podman[144553]: 2024-04-05 03:19:27.347851042 +0000 UTC m=+0.660251833 container remove cfa7f55e5875d8d4118b71ad51d32aa910445ea51c4b9649521174c51eedaf52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-05T03:19:28.066 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:27 smithi033 podman[144643]: 2024-04-05 03:19:27.765216084 +0000 UTC m=+0.094713388 container create 1a5fd148e5c64c360e3c11edaaf748c05447397a350d621b5b0517d78faa3e20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, RELEASE=squid-0028525, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-05T03:19:28.066 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:27 smithi033 podman[144643]: 2024-04-05 03:19:27.688499795 +0000 UTC m=+0.017997120 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:19:28.066 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:27 smithi033 podman[144643]: 2024-04-05 03:19:27.992681416 +0000 UTC m=+0.322178722 container init 1a5fd148e5c64c360e3c11edaaf748c05447397a350d621b5b0517d78faa3e20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240403, io.buildah.version=1.33.5) 2024-04-05T03:19:28.066 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:27 smithi033 podman[144643]: 2024-04-05 03:19:27.995724136 +0000 UTC m=+0.325221445 container start 1a5fd148e5c64c360e3c11edaaf748c05447397a350d621b5b0517d78faa3e20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:19:28.479 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:28 smithi033 podman[144643]: 2024-04-05 03:19:28.067433712 +0000 UTC m=+0.396931037 container attach 1a5fd148e5c64c360e3c11edaaf748c05447397a350d621b5b0517d78faa3e20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-05T03:19:28.479 INFO:journalctl@ceph.osd.2.smithi033.stdout:Apr 05 03:19:28 smithi033 podman[144643]: 2024-04-05 03:19:28.288336201 +0000 UTC m=+0.617833507 container died 1a5fd148e5c64c360e3c11edaaf748c05447397a350d621b5b0517d78faa3e20 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:19:28.868 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.2.service' 2024-04-05T03:19:28.912 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:28.913 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-05T03:19:28.913 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-05T03:19:28.913 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.3 2024-04-05T03:19:29.229 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:29 smithi033 systemd[1]: Stopping Ceph osd.3 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:29.729 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[124226]: 2024-04-05T03:19:29.230+0000 7fa4e8e60700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:29.730 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[124226]: 2024-04-05T03:19:29.231+0000 7fa4e8e60700 -1 osd.3 108 *** Got signal Terminated *** 2024-04-05T03:19:29.730 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:29 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3[124226]: 2024-04-05T03:19:29.231+0000 7fa4e8e60700 -1 osd.3 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:19:32.030 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:31 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:31.674+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:33.000 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:32 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:32.707+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:34.027 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:33 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:33.753+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:34.027 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:33 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:33.868+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:34.578 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:34.278Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:19:34.578 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:34.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:19:34.578 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:34 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:34.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:19:34.856 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:34 smithi033 podman[144749]: 2024-04-05 03:19:34.579867438 +0000 UTC m=+5.434046153 container died ca165607e43d885feda76099e5a64c37447195b253ba64614b0848fa29f0e2c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-05T03:19:35.027 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:34 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:34.715+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:35.027 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:34 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:34.840+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:35.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:34 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:34.724+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:35.206 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 podman[144749]: 2024-04-05 03:19:35.023298409 +0000 UTC m=+5.877477132 container cleanup ca165607e43d885feda76099e5a64c37447195b253ba64614b0848fa29f0e2c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-05T03:19:35.206 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 bash[144749]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3 2024-04-05T03:19:35.479 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 podman[144770]: 2024-04-05 03:19:35.207910674 +0000 UTC m=+0.618840260 container remove ca165607e43d885feda76099e5a64c37447195b253ba64614b0848fa29f0e2c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:19:35.959 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 podman[144860]: 2024-04-05 03:19:35.540937459 +0000 UTC m=+0.017417279 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:19:35.959 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 podman[144860]: 2024-04-05 03:19:35.641972258 +0000 UTC m=+0.118452120 container create c53028a36300e3ed752e8c12deab6baecaeaaec98e8f5cbedc6ea525d2c13174 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-05T03:19:35.959 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 podman[144860]: 2024-04-05 03:19:35.875959168 +0000 UTC m=+0.352438991 container init c53028a36300e3ed752e8c12deab6baecaeaaec98e8f5cbedc6ea525d2c13174 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, org.label-schema.schema-version=1.0) 2024-04-05T03:19:35.959 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 podman[144860]: 2024-04-05 03:19:35.87893092 +0000 UTC m=+0.355410746 container start c53028a36300e3ed752e8c12deab6baecaeaaec98e8f5cbedc6ea525d2c13174 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-0028525, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-05T03:19:35.959 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:35 smithi033 podman[144860]: 2024-04-05 03:19:35.960802168 +0000 UTC m=+0.437281990 container attach c53028a36300e3ed752e8c12deab6baecaeaaec98e8f5cbedc6ea525d2c13174 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-05T03:19:36.027 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:35.874+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:36.027 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:35.667+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:36.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:35 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:35.770+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:36.229 INFO:journalctl@ceph.osd.3.smithi033.stdout:Apr 05 03:19:36 smithi033 podman[144860]: 2024-04-05 03:19:36.170263109 +0000 UTC m=+0.646742929 container died c53028a36300e3ed752e8c12deab6baecaeaaec98e8f5cbedc6ea525d2c13174 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, ceph=True) 2024-04-05T03:19:36.650 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:36 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:36.400+0000 7f2188118700 -1 osd.4 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:09.995872+0000 front 2024-04-05T03:19:09.995762+0000 (oldest deadline 2024-04-05T03:19:35.895729+0000) 2024-04-05T03:19:36.730 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.3.service' 2024-04-05T03:19:36.774 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:36.774 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-05T03:19:36.775 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-05T03:19:36.775 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.4 2024-04-05T03:19:37.027 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:36 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:36.897+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:37.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:36 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:36.725+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:37.028 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:36 smithi151 systemd[1]: Stopping Ceph osd.4 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:37.028 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:36 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:36.651+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:37.350 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:37.072+0000 7f218a317700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:37.350 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:37.072+0000 7f218a317700 -1 osd.4 108 *** Got signal Terminated *** 2024-04-05T03:19:37.350 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:37.072+0000 7f218a317700 -1 osd.4 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:19:37.350 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:37.352+0000 7f2188118700 -1 osd.4 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:09.995872+0000 front 2024-04-05T03:19:09.995762+0000 (oldest deadline 2024-04-05T03:19:35.895729+0000) 2024-04-05T03:19:37.932 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:37.614+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:37.933 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:37.714+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:38.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:37 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:37.934+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:38.722 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:38.354+0000 7f2188118700 -1 osd.4 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:09.995872+0000 front 2024-04-05T03:19:09.995762+0000 (oldest deadline 2024-04-05T03:19:35.895729+0000) 2024-04-05T03:19:38.723 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:38.568+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:39.027 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:38.962+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:39.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:38 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:38.724+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:39.693 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:39.351+0000 7f2188118700 -1 osd.4 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:09.995872+0000 front 2024-04-05T03:19:09.995762+0000 (oldest deadline 2024-04-05T03:19:35.895729+0000) 2024-04-05T03:19:39.694 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:39.541+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:39.694 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:39.541+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:39.972 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:39.695+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:40.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:39.974+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:40.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:39 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:39.974+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:40.724 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:40 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:40.348+0000 7f2188118700 -1 osd.4 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:09.995872+0000 front 2024-04-05T03:19:09.995762+0000 (oldest deadline 2024-04-05T03:19:35.895729+0000) 2024-04-05T03:19:40.725 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:40 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:40.550+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:40.725 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:40 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:40.550+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:41.009 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:40 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:40.726+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:41.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:41.010+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:41.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:41.010+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:41.698 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:41.359+0000 7f2188118700 -1 osd.4 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:09.995872+0000 front 2024-04-05T03:19:09.995762+0000 (oldest deadline 2024-04-05T03:19:35.895729+0000) 2024-04-05T03:19:41.699 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4[37714]: 2024-04-05T03:19:41.359+0000 7f2188118700 -1 osd.4 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:15.896438+0000 front 2024-04-05T03:19:15.896339+0000 (oldest deadline 2024-04-05T03:19:41.196301+0000) 2024-04-05T03:19:41.699 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:41.504+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:41.699 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:41.504+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:42.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:41 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:41.700+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:42.392 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:42.045+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:42.393 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:42.045+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:42.652 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:42 smithi151 podman[99068]: 2024-04-05 03:19:42.394458537 +0000 UTC m=+5.421554617 container died b7f73016d7ed83e6b04555e1cd81d4c888b385c3601d70ce533c7f9cf91c3162 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, ceph=True) 2024-04-05T03:19:42.652 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:42.546+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:42.653 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:42.547+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:42.997 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:42 smithi151 podman[99068]: 2024-04-05 03:19:42.7730545 +0000 UTC m=+5.800150638 container cleanup b7f73016d7ed83e6b04555e1cd81d4c888b385c3601d70ce533c7f9cf91c3162 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-05T03:19:42.998 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:42 smithi151 bash[99068]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4 2024-04-05T03:19:42.998 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:42.654+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:42.998 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:42.654+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:43.277 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:43 smithi151 podman[99085]: 2024-04-05 03:19:43.057880923 +0000 UTC m=+0.654339540 container remove b7f73016d7ed83e6b04555e1cd81d4c888b385c3601d70ce533c7f9cf91c3162 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS) 2024-04-05T03:19:43.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:42.998+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:43.278 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:42 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:42.998+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:43.628 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:43.501+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:43.629 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:43.501+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:43.629 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:43.629+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:43.629 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:43.630+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:43.630 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:43 smithi151 podman[99177]: 2024-04-05 03:19:43.375496544 +0000 UTC m=+0.114889512 container create df6fc22e40e3a1624629936ffcf6ea30e72b4e08e372e2f3eb98c6eb5181ad14 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:19:43.630 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:43 smithi151 podman[99177]: 2024-04-05 03:19:43.278538184 +0000 UTC m=+0.017931151 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:19:43.630 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:43 smithi151 podman[99177]: 2024-04-05 03:19:43.595143993 +0000 UTC m=+0.334536959 container init df6fc22e40e3a1624629936ffcf6ea30e72b4e08e372e2f3eb98c6eb5181ad14 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-04-05T03:19:43.630 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:43 smithi151 podman[99177]: 2024-04-05 03:19:43.598413282 +0000 UTC m=+0.337806251 container start df6fc22e40e3a1624629936ffcf6ea30e72b4e08e372e2f3eb98c6eb5181ad14 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240403) 2024-04-05T03:19:43.884 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:43 smithi151 podman[99177]: 2024-04-05 03:19:43.686769718 +0000 UTC m=+0.426162706 container attach df6fc22e40e3a1624629936ffcf6ea30e72b4e08e372e2f3eb98c6eb5181ad14 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, ceph=True) 2024-04-05T03:19:44.196 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:43.969+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:44.196 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:43 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:43.969+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:44.196 INFO:journalctl@ceph.osd.4.smithi151.stdout:Apr 05 03:19:43 smithi151 podman[99177]: 2024-04-05 03:19:43.886979013 +0000 UTC m=+0.626371987 container died df6fc22e40e3a1624629936ffcf6ea30e72b4e08e372e2f3eb98c6eb5181ad14 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:19:44.464 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.4.service' 2024-04-05T03:19:44.507 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:44.507 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-05T03:19:44.507 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-05T03:19:44.508 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.5 2024-04-05T03:19:44.729 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:44.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:19:44.729 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:44.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:19:44.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:44 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:44.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:19:44.736 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:44.673+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:44.737 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:44.673+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:44.737 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:44.476+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:44.737 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:44.476+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:44.737 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:44 smithi151 systemd[1]: Stopping Ceph osd.5 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:45.027 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:44.858+0000 7fe57fd76700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:45.028 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:44.858+0000 7fe57fd76700 -1 osd.5 108 *** Got signal Terminated *** 2024-04-05T03:19:45.028 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:44.858+0000 7fe57fd76700 -1 osd.5 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:19:45.028 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:44.928+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:45.028 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:44 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:44.928+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:45.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:45.660+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:45.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:45.660+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:45.777 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:45.470+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:45.778 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:45.470+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:46.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:45.930+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:46.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:45 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:45.930+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:46.654 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:46.433+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:46.655 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:46.433+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:46.655 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:19:46.348Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-04-05T03:19:46.655 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:19:46.348Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-04-05T03:19:46.655 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:19:46.348Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-04-05T03:19:46.655 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:19:46.348Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-04-05T03:19:46.656 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:19:46.350Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-04-05T03:19:46.942 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:46.655+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:46.943 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:46.655+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:47.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:46.943+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:47.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:46 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:46.943+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:47.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:47.636+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:47.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:47.636+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:47.778 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:47.479+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:47.778 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:47.479+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:48.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:47.900+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:48.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:47.900+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:48.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:47 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:47.900+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:48.777 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:48.521+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:48.777 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:48.521+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:48.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:48.675+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:48.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:48.675+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:48.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:48.675+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:49.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:48.881+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:49.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:48.881+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:49.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:48 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:48.881+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:49.844 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:49.567+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:06.212649+0000 front 2024-04-05T03:19:06.212621+0000 (oldest deadline 2024-04-05T03:19:31.512578+0000) 2024-04-05T03:19:49.844 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:49.567+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.414041+0000 front 2024-04-05T03:19:17.413979+0000 (oldest deadline 2024-04-05T03:19:39.113917+0000) 2024-04-05T03:19:49.844 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5[43777]: 2024-04-05T03:19:49.567+0000 7fe579b6f700 -1 osd.5 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:25.316423+0000 front 2024-04-05T03:19:25.316327+0000 (oldest deadline 2024-04-05T03:19:49.416297+0000) 2024-04-05T03:19:49.845 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:49.652+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:49.845 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:49.652+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:49.845 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:49.652+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:50.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:49.844+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:50.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:49.844+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:50.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:49 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:49.844+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:50.567 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:50 smithi151 podman[99283]: 2024-04-05 03:19:50.306605281 +0000 UTC m=+5.562598866 container died e2607db1cdd0e1c48efc9a0a4527f1dcc8690542c0074241975f5bc63b5f46a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-05T03:19:50.859 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:50 smithi151 podman[99283]: 2024-04-05 03:19:50.651418447 +0000 UTC m=+5.907412040 container cleanup e2607db1cdd0e1c48efc9a0a4527f1dcc8690542c0074241975f5bc63b5f46a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-05T03:19:50.859 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:50 smithi151 bash[99283]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5 2024-04-05T03:19:50.860 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:50 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:50.805+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:50.860 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:50 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:50.805+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:50.860 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:50 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:50.805+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:50.860 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:50 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:50.648+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:50.861 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:50 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:50.648+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:50.861 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:50 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:50.649+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:51.267 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:50 smithi151 podman[99312]: 2024-04-05 03:19:50.861359017 +0000 UTC m=+0.549161817 container remove e2607db1cdd0e1c48efc9a0a4527f1dcc8690542c0074241975f5bc63b5f46a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:19:51.527 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:51 smithi151 podman[99406]: 2024-04-05 03:19:51.173140849 +0000 UTC m=+0.018111804 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:19:51.527 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:51 smithi151 podman[99406]: 2024-04-05 03:19:51.35407382 +0000 UTC m=+0.199044746 container create 287a55f8a3c6d4c7049b62ddadfa9592ac4c479e9d625a425f20f471535e0225 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD) 2024-04-05T03:19:51.854 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:51 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:51.804+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:51.855 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:51 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:51.804+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:51.855 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:51 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:51.805+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:51.855 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:51 smithi151 podman[99406]: 2024-04-05 03:19:51.564965355 +0000 UTC m=+0.409936283 container init 287a55f8a3c6d4c7049b62ddadfa9592ac4c479e9d625a425f20f471535e0225 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:19:51.855 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:51 smithi151 podman[99406]: 2024-04-05 03:19:51.567927756 +0000 UTC m=+0.412898684 container start 287a55f8a3c6d4c7049b62ddadfa9592ac4c479e9d625a425f20f471535e0225 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-05T03:19:51.855 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:51 smithi151 podman[99406]: 2024-04-05 03:19:51.656639689 +0000 UTC m=+0.501610626 container attach 287a55f8a3c6d4c7049b62ddadfa9592ac4c479e9d625a425f20f471535e0225 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403) 2024-04-05T03:19:51.856 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:51 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:51.621+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:51.856 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:51 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:51.621+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:51.856 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:51 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:51.621+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:52.157 INFO:journalctl@ceph.osd.5.smithi151.stdout:Apr 05 03:19:51 smithi151 podman[99406]: 2024-04-05 03:19:51.857152537 +0000 UTC m=+0.702123479 container died 287a55f8a3c6d4c7049b62ddadfa9592ac4c479e9d625a425f20f471535e0225 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=) 2024-04-05T03:19:52.458 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.5.service' 2024-04-05T03:19:52.503 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:19:52.503 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-05T03:19:52.504 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-05T03:19:52.504 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.6 2024-04-05T03:19:52.777 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:52 smithi151 systemd[1]: Stopping Ceph osd.6 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:19:52.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:52.632+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:52.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:52.632+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:52.778 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:52.632+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:53.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:52.837+0000 7f144eb3a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:19:53.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:52.837+0000 7f144eb3a700 -1 osd.6 108 *** Got signal Terminated *** 2024-04-05T03:19:53.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:52.837+0000 7f144eb3a700 -1 osd.6 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:19:53.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:52.838+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:53.278 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:52.838+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:53.278 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:52 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:52.838+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:53.865 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:53.613+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:53.865 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:53.613+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:53.865 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:53.613+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:54.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:53.866+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:54.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:53.866+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:54.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:53 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:53.866+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:54.729 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:54 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:54.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:19:54.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:54 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:54.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:19:54.730 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:19:54 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:19:54.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:19:54.910 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:54.644+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:54.910 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:54.644+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:54.910 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:54.644+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:55.185 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:54.910+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:55.186 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:54.910+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:55.186 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:54 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:54.910+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:55.932 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:55 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:55.613+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:55.932 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:55 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:55.613+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:55.932 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:55 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:55.613+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:55.932 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:55 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:55.613+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:19:56.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:55 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:55.929+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:56.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:55 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:55.929+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:56.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:55 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:55.929+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:56.894 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:56.593+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:56.895 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:56.593+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:56.895 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:56.593+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:56.895 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:56.593+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:19:57.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:56.895+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:10.221140+0000 front 2024-04-05T03:19:10.221047+0000 (oldest deadline 2024-04-05T03:19:33.720935+0000) 2024-04-05T03:19:57.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:56.895+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:13.721610+0000 front 2024-04-05T03:19:13.721523+0000 (oldest deadline 2024-04-05T03:19:39.621272+0000) 2024-04-05T03:19:57.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:56.895+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:23.122384+0000 front 2024-04-05T03:19:23.122219+0000 (oldest deadline 2024-04-05T03:19:47.822105+0000) 2024-04-05T03:19:57.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:56 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6[49783]: 2024-04-05T03:19:56.895+0000 7f144a136700 -1 osd.6 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.423561+0000 front 2024-04-05T03:19:32.423625+0000 (oldest deadline 2024-04-05T03:19:56.523554+0000) 2024-04-05T03:19:58.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:57.580+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:58.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:57.580+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:58.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:57.580+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:58.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:57 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:57.580+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:19:58.554 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:58 smithi151 podman[99514]: 2024-04-05 03:19:58.302388054 +0000 UTC m=+5.578972598 container died 2063f47e19f26713f2eb1d2ee1c73fc256e214ab2043acd8476c4ef2fabd9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-05T03:19:58.879 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:58 smithi151 podman[99514]: 2024-04-05 03:19:58.646514715 +0000 UTC m=+5.923099259 container cleanup 2063f47e19f26713f2eb1d2ee1c73fc256e214ab2043acd8476c4ef2fabd9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403) 2024-04-05T03:19:58.880 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:58 smithi151 bash[99514]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6 2024-04-05T03:19:58.880 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:58.578+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:19:58.880 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:58.578+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:19:58.880 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:58.578+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:19:58.880 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:58 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:58.578+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:19:59.277 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:58 smithi151 podman[99534]: 2024-04-05 03:19:58.881344336 +0000 UTC m=+0.574568025 container remove 2063f47e19f26713f2eb1d2ee1c73fc256e214ab2043acd8476c4ef2fabd9642 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-04-05T03:19:59.574 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:59 smithi151 podman[99624]: 2024-04-05 03:19:59.192642635 +0000 UTC m=+0.018052106 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:19:59.574 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:59 smithi151 podman[99624]: 2024-04-05 03:19:59.2904045 +0000 UTC m=+0.115813971 container create 4008b1c85cff7f911605d489a0ba6f06c35980965accc0a430187d551f21aef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:19:59.575 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:59 smithi151 podman[99624]: 2024-04-05 03:19:59.493370694 +0000 UTC m=+0.318780170 container init 4008b1c85cff7f911605d489a0ba6f06c35980965accc0a430187d551f21aef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-05T03:19:59.575 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:59 smithi151 podman[99624]: 2024-04-05 03:19:59.496553849 +0000 UTC m=+0.321963324 container start 4008b1c85cff7f911605d489a0ba6f06c35980965accc0a430187d551f21aef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6-deactivate, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:19:59.575 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:59 smithi151 podman[99624]: 2024-04-05 03:19:59.57639212 +0000 UTC m=+0.401801599 container attach 4008b1c85cff7f911605d489a0ba6f06c35980965accc0a430187d551f21aef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6-deactivate, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-05T03:20:00.027 INFO:journalctl@ceph.osd.6.smithi151.stdout:Apr 05 03:19:59 smithi151 podman[99624]: 2024-04-05 03:19:59.776988132 +0000 UTC m=+0.602397603 container died 4008b1c85cff7f911605d489a0ba6f06c35980965accc0a430187d551f21aef6 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-05T03:20:00.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:59.611+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:20:00.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:59.611+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:20:00.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:59.611+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:20:00.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:19:59 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:19:59.611+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:20:00.366 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.6.service' 2024-04-05T03:20:00.409 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:20:00.410 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-05T03:20:00.410 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-05T03:20:00.410 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.7 2024-04-05T03:20:00.621 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 systemd[1]: Stopping Ceph osd.7 for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:20:01.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:00.621+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:20:01.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:00.621+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:20:01.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:00.621+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:20:01.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:00.621+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:20:01.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:00.723+0000 7ff0d29b8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:20:01.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:00.723+0000 7ff0d29b8700 -1 osd.7 108 *** Got signal Terminated *** 2024-04-05T03:20:01.029 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:00 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:00.723+0000 7ff0d29b8700 -1 osd.7 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:20:02.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:01.581+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:20:02.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:01.581+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:20:02.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:01.581+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:20:02.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:01 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:01.581+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:20:03.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:02.579+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:20:03.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:02.579+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:20:03.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:02.579+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:20:03.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:02 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:02.579+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:20:04.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:03 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:03.562+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:20:04.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:03 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:03.562+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:20:04.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:03 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:03.562+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:20:04.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:03 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:03.562+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:20:04.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:03 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:03.562+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.151:6804 osd.4 since back 2024-04-05T03:19:36.690682+0000 front 2024-04-05T03:19:36.690707+0000 (oldest deadline 2024-04-05T03:20:02.590640+0000) 2024-04-05T03:20:04.640 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:20:04.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:20:04.641 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:20:04.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi033.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-04-05T03:20:04.641 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:04 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:20:04.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi151.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.151:8443: connect: connection refused" 2024-04-05T03:20:05.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:04.563+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:20:05.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:04.563+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:20:05.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:04.563+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:20:05.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:04.563+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:20:05.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:04 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:04.563+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.151:6804 osd.4 since back 2024-04-05T03:19:36.690682+0000 front 2024-04-05T03:19:36.690707+0000 (oldest deadline 2024-04-05T03:20:02.590640+0000) 2024-04-05T03:20:06.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:05.546+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-04-05T03:19:08.687421+0000 front 2024-04-05T03:19:08.687488+0000 (oldest deadline 2024-04-05T03:19:33.987256+0000) 2024-04-05T03:20:06.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:05.546+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-04-05T03:19:17.488101+0000 front 2024-04-05T03:19:17.488130+0000 (oldest deadline 2024-04-05T03:19:42.188055+0000) 2024-04-05T03:20:06.027 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:05.546+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-04-05T03:19:26.289054+0000 front 2024-04-05T03:19:26.289102+0000 (oldest deadline 2024-04-05T03:19:48.588919+0000) 2024-04-05T03:20:06.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:05.546+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-04-05T03:19:32.089851+0000 front 2024-04-05T03:19:32.089812+0000 (oldest deadline 2024-04-05T03:19:54.989703+0000) 2024-04-05T03:20:06.028 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:05 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7[55484]: 2024-04-05T03:20:05.546+0000 7ff0ce7b5700 -1 osd.7 108 heartbeat_check: no reply from 172.21.15.151:6804 osd.4 since back 2024-04-05T03:19:36.690682+0000 front 2024-04-05T03:19:36.690707+0000 (oldest deadline 2024-04-05T03:20:02.590640+0000) 2024-04-05T03:20:06.527 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:06 smithi151 podman[99731]: 2024-04-05 03:20:06.254376397 +0000 UTC m=+5.620372411 container died 164624ba7a6da29e3c12fcc8994cd8bddc3d5172fec8a18580130687ae9f7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7, RELEASE=quincy-6fda41f, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5) 2024-04-05T03:20:06.881 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:06 smithi151 podman[99731]: 2024-04-05 03:20:06.623878503 +0000 UTC m=+5.989874515 container cleanup 164624ba7a6da29e3c12fcc8994cd8bddc3d5172fec8a18580130687ae9f7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403) 2024-04-05T03:20:06.882 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:06 smithi151 bash[99731]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7 2024-04-05T03:20:07.232 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:06 smithi151 podman[99761]: 2024-04-05 03:20:06.883691666 +0000 UTC m=+0.623865538 container remove 164624ba7a6da29e3c12fcc8994cd8bddc3d5172fec8a18580130687ae9f7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7, RELEASE=quincy-6fda41f, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True) 2024-04-05T03:20:07.526 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:07 smithi151 podman[99854]: 2024-04-05 03:20:07.137437646 +0000 UTC m=+0.017699632 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:20:07.527 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:07 smithi151 podman[99854]: 2024-04-05 03:20:07.242846895 +0000 UTC m=+0.123108865 container create 5bc978d481938a075ee95233310f2e41f4c5dfcafc5109d587a3abd01e6969ff (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f) 2024-04-05T03:20:07.527 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:07 smithi151 podman[99854]: 2024-04-05 03:20:07.445504913 +0000 UTC m=+0.325766900 container init 5bc978d481938a075ee95233310f2e41f4c5dfcafc5109d587a3abd01e6969ff (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-05T03:20:07.527 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:07 smithi151 podman[99854]: 2024-04-05 03:20:07.449079636 +0000 UTC m=+0.329341606 container start 5bc978d481938a075ee95233310f2e41f4c5dfcafc5109d587a3abd01e6969ff (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-04-05T03:20:07.527 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:07 smithi151 podman[99854]: 2024-04-05 03:20:07.528587896 +0000 UTC m=+0.408849870 container attach 5bc978d481938a075ee95233310f2e41f4c5dfcafc5109d587a3abd01e6969ff (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, ceph=True, GIT_CLEAN=True, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-05T03:20:07.777 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:07 smithi151 podman[99854]: 2024-04-05 03:20:07.733112163 +0000 UTC m=+0.613374135 container died 5bc978d481938a075ee95233310f2e41f4c5dfcafc5109d587a3abd01e6969ff (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403) 2024-04-05T03:20:08.291 INFO:journalctl@ceph.osd.7.smithi151.stdout:Apr 05 03:20:08 smithi151 podman[99880]: 2024-04-05 03:20:08.273613346 +0000 UTC m=+0.535166316 container remove 5bc978d481938a075ee95233310f2e41f4c5dfcafc5109d587a3abd01e6969ff (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:20:08.296 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@osd.7.service' 2024-04-05T03:20:08.338 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:20:08.339 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-05T03:20:08.339 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-05T03:20:08.339 DEBUG:teuthology.orchestra.run.smithi151:> sudo systemctl stop ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a 2024-04-05T03:20:08.666 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 systemd[1]: Stopping Ceph prometheus.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:20:08.933 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:20:08.934 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T03:20:08.934 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T03:20:08.934 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T03:20:08.934 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T03:20:08.934 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T03:20:08.935 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T03:20:08.935 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T03:20:08.935 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T03:20:08.935 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T03:20:08.935 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T03:20:08.935 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a[77871]: ts=2024-04-05T03:20:08.668Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T03:20:08.935 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:08 smithi151 podman[99959]: 2024-04-05 03:20:08.679378021 +0000 UTC m=+0.120374815 container died c6c02f0b66d84945d2594a63bc6d0f5b732171ed14c8458fc610f08aa444f81a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:09.277 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:09 smithi151 podman[99959]: 2024-04-05 03:20:09.036294613 +0000 UTC m=+0.477291401 container cleanup c6c02f0b66d84945d2594a63bc6d0f5b732171ed14c8458fc610f08aa444f81a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:09.277 INFO:journalctl@ceph.prometheus.a.smithi151.stdout:Apr 05 03:20:09 smithi151 bash[99959]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T03:20:09.407 DEBUG:teuthology.orchestra.run.smithi151:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@prometheus.a.service' 2024-04-05T03:20:09.443 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:20:09.443 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-05T03:20:09.445 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-05T03:20:11.204 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:10 smithi033 systemd[1]: Stopping Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:20:11.479 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:11 smithi033 ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a[87526]: ts=2024-04-05T03:20:11.205Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:20:11.479 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:11 smithi033 podman[145267]: 2024-04-05 03:20:11.246958037 +0000 UTC m=+0.142910854 container died a9cf7c76c5199cfdd4bef93e305487b070a336fd4caaa70829f6fe9a947b387b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:11.847 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:11 smithi033 podman[145267]: 2024-04-05 03:20:11.689354748 +0000 UTC m=+0.585307565 container cleanup a9cf7c76c5199cfdd4bef93e305487b070a336fd4caaa70829f6fe9a947b387b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:11.847 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:11 smithi033 bash[145267]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-05T03:20:12.194 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:11 smithi033 podman[145279]: 2024-04-05 03:20:11.84903424 +0000 UTC m=+0.594016665 container remove a9cf7c76c5199cfdd4bef93e305487b070a336fd4caaa70829f6fe9a947b387b (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:12.195 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:11 smithi033 podman[145279]: 2024-04-05 03:20:11.966597082 +0000 UTC m=+0.711579465 volume remove 58cd73f5c8d771e0d4f95555e9fb2a7b64f5aff2194792b055999d26d3ad9488 2024-04-05T03:20:12.479 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:12 smithi033 systemd[1]: Stopping Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:20:12.479 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:12 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-05T03:20:12.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:12 smithi033 systemd[1]: Stopped Ceph alertmanager.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:20:12.480 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Apr 05 03:20:12 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@alertmanager.a.service: Consumed 1.677s CPU time. 2024-04-05T03:20:12.979 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:12 smithi033 podman[145390]: 2024-04-05 03:20:12.647006225 +0000 UTC m=+0.118332009 container died 6da1bdc228ef38d7f121f391c9f2b69ef55931ecc6207b5e821b57a92c19fbe8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:13.352 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:13 smithi033 podman[145390]: 2024-04-05 03:20:13.077494148 +0000 UTC m=+0.548819911 container cleanup 6da1bdc228ef38d7f121f391c9f2b69ef55931ecc6207b5e821b57a92c19fbe8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:13.353 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:13 smithi033 bash[145390]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a 2024-04-05T03:20:13.639 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:13 smithi033 podman[145402]: 2024-04-05 03:20:13.354164471 +0000 UTC m=+0.712044224 container remove 6da1bdc228ef38d7f121f391c9f2b69ef55931ecc6207b5e821b57a92c19fbe8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:20:13.639 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:13 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:20:13.979 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:13 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-05T03:20:13.979 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:13 smithi033 systemd[1]: Stopped Ceph node-exporter.a for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:20:13.979 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Apr 05 03:20:13 smithi033 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.a.service: Consumed 2.202s CPU time. 2024-04-05T03:20:21.475 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-05T03:20:23.527 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:23 smithi151 systemd[1]: Stopping Ceph node-exporter.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4... 2024-04-05T03:20:23.527 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:23 smithi151 podman[100405]: 2024-04-05 03:20:23.390290891 +0000 UTC m=+0.086729195 container died 18b78d165e692025c44126f37f53bc053d3dddf0b99f87a2d995bf6b390cf39a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:20:23.911 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:23 smithi151 podman[100405]: 2024-04-05 03:20:23.696282714 +0000 UTC m=+0.392721012 container cleanup 18b78d165e692025c44126f37f53bc053d3dddf0b99f87a2d995bf6b390cf39a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:20:23.912 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:23 smithi151 bash[100405]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b 2024-04-05T03:20:24.221 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:23 smithi151 podman[100419]: 2024-04-05 03:20:23.914037657 +0000 UTC m=+0.524480889 container remove 18b78d165e692025c44126f37f53bc053d3dddf0b99f87a2d995bf6b390cf39a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:20:24.222 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:23 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:20:24.222 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:24 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-05T03:20:24.222 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:24 smithi151 systemd[1]: Stopped Ceph node-exporter.b for f4bb5524-f2f7-11ee-b649-cb9ed24678a4. 2024-04-05T03:20:24.222 INFO:journalctl@ceph.node-exporter.b.smithi151.stdout:Apr 05 03:20:24 smithi151 systemd[1]: ceph-f4bb5524-f2f7-11ee-b649-cb9ed24678a4@node-exporter.b.service: Consumed 2.049s CPU time. 2024-04-05T03:20:28.358 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:20:28.393 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:20:28.422 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-05T03:20:28.424 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239/remote/smithi033/crash 2024-04-05T03:20:28.425 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/crash -- . 2024-04-05T03:20:28.479 INFO:teuthology.orchestra.run.smithi033.stderr:tar: /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-05T03:20:28.479 INFO:teuthology.orchestra.run.smithi033.stderr:tar: Error is not recoverable: exiting now 2024-04-05T03:20:28.481 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239/remote/smithi151/crash 2024-04-05T03:20:28.482 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/crash -- . 2024-04-05T03:20:28.543 INFO:teuthology.orchestra.run.smithi151.stderr:tar: /var/lib/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-05T03:20:28.543 INFO:teuthology.orchestra.run.smithi151.stderr:tar: Error is not recoverable: exiting now 2024-04-05T03:20:28.544 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-05T03:20:28.545 DEBUG:teuthology.orchestra.run.smithi033:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-05T03:20:28.575 INFO:teuthology.orchestra.run.smithi033.stdout:1712285869.2417617 mon.a (mon.0) 172 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-05T03:20:28.576 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-05T03:20:28.576 INFO:tasks.cephadm:Compressing logs... 2024-04-05T03:20:28.576 DEBUG:teuthology.orchestra.run.smithi033:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-05T03:20:28.618 DEBUG:teuthology.orchestra.run.smithi151:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-05T03:20:28.644 INFO:teuthology.orchestra.run.smithi033.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/cephadm.log 2024-04-05T03:20:28.644 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-04-05T03:20:28.644 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.a.log 2024-04-05T03:20:28.655 INFO:teuthology.orchestra.run.smithi151.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-05T03:20:28.655 INFO:teuthology.orchestra.run.smithi151.stderr:: No such file or directory 2024-04-05T03:20:28.656 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.6.log 2024-04-05T03:20:28.656 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.audit.log 2024-04-05T03:20:28.656 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi151.cxpowy.log 2024-04-05T03:20:28.656 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.4.log 2024-04-05T03:20:28.656 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.7.log 2024-04-05T03:20:28.656 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi151.cxpowy.log: 62.6% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi151.cxpowy.log.gz 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.5.log 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mgr.x.log 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.cephadm.log 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.log 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.cephadm.log: 83.3% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr:gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-volume.log 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr: 89.6% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.audit.log.gz 2024-04-05T03:20:28.657 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.b.log 2024-04-05T03:20:28.658 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-volume.log: 87.6% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.log.gz 2024-04-05T03:20:28.658 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.audit.log 2024-04-05T03:20:28.658 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.1.log 2024-04-05T03:20:28.658 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi033.rmzwzs.log 2024-04-05T03:20:28.658 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.3.log 2024-04-05T03:20:28.658 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi033.rmzwzs.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mgr.y.log 2024-04-05T03:20:28.658 INFO:teuthology.orchestra.run.smithi033.stderr: 81.9% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi033.rmzwzs.log.gz 2024-04-05T03:20:28.659 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.0.log 2024-04-05T03:20:28.659 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.cephadm.log 2024-04-05T03:20:28.659 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.log 2024-04-05T03:20:28.659 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.cephadm.log: 90.0% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-05T03:20:28.659 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.2.log 2024-04-05T03:20:28.659 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.log: 93.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-05T03:20:28.659 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-volume.log 2024-04-05T03:20:28.660 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.2.log: 93.6% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.audit.log.gz 2024-04-05T03:20:28.660 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.c.log 2024-04-05T03:20:28.660 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-volume.log: 92.7% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph.log.gz 2024-04-05T03:20:28.660 INFO:teuthology.orchestra.run.smithi151.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.b.log: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-05T03:20:28.665 INFO:teuthology.orchestra.run.smithi151.stderr: 91.3% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-volume.log.gz 2024-04-05T03:20:28.673 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.c.log: 91.0% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-volume.log.gz 2024-04-05T03:20:28.695 INFO:teuthology.orchestra.run.smithi151.stderr: 89.6% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mgr.x.log.gz 2024-04-05T03:20:28.968 INFO:teuthology.orchestra.run.smithi151.stderr: 92.3% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.b.log.gz 2024-04-05T03:20:29.383 INFO:teuthology.orchestra.run.smithi033.stderr: 92.2% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.c.log.gz 2024-04-05T03:20:29.383 INFO:teuthology.orchestra.run.smithi033.stderr: 88.7% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mgr.y.log.gz 2024-04-05T03:20:29.383 INFO:teuthology.orchestra.run.smithi033.stderr: 93.3% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-05T03:20:29.383 INFO:teuthology.orchestra.run.smithi033.stderr: 92.9% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-05T03:20:29.384 INFO:teuthology.orchestra.run.smithi151.stderr: 92.7% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-05T03:20:29.384 INFO:teuthology.orchestra.run.smithi151.stderr: 93.3% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-05T03:20:29.384 INFO:teuthology.orchestra.run.smithi151.stderr: 93.1% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-05T03:20:29.384 INFO:teuthology.orchestra.run.smithi151.stderr: 93.2% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-05T03:20:29.384 INFO:teuthology.orchestra.run.smithi151.stderr: 2024-04-05T03:20:29.384 INFO:teuthology.orchestra.run.smithi151.stderr:real 0m0.689s 2024-04-05T03:20:29.385 INFO:teuthology.orchestra.run.smithi151.stderr:user 0m2.652s 2024-04-05T03:20:29.385 INFO:teuthology.orchestra.run.smithi151.stderr:sys 0m0.123s 2024-04-05T03:20:29.387 INFO:teuthology.orchestra.run.smithi033.stderr: 93.2% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-05T03:20:29.505 INFO:teuthology.orchestra.run.smithi033.stderr: 93.6% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-05T03:20:29.591 INFO:teuthology.orchestra.run.smithi033.stderr: 90.6% -- replaced with /var/log/ceph/f4bb5524-f2f7-11ee-b649-cb9ed24678a4/ceph-mon.a.log.gz 2024-04-05T03:20:29.594 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-04-05T03:20:29.594 INFO:teuthology.orchestra.run.smithi033.stderr:real 0m0.963s 2024-04-05T03:20:29.594 INFO:teuthology.orchestra.run.smithi033.stderr:user 0m4.461s 2024-04-05T03:20:29.595 INFO:teuthology.orchestra.run.smithi033.stderr:sys 0m0.174s 2024-04-05T03:20:29.595 INFO:tasks.cephadm:Archiving logs... 2024-04-05T03:20:29.596 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239/remote/smithi033/log 2024-04-05T03:20:29.807 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-05T03:20:30.685 DEBUG:teuthology.misc:Transferring archived files from smithi151:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239/remote/smithi151/log 2024-04-05T03:20:30.686 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-05T03:20:31.105 INFO:tasks.cephadm:Removing cluster... 2024-04-05T03:20:31.106 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 --force 2024-04-05T03:20:31.595 DEBUG:teuthology.orchestra.run.smithi151:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f4bb5524-f2f7-11ee-b649-cb9ed24678a4 --force 2024-04-05T03:20:32.079 INFO:tasks.cephadm:Removing cephadm ... 2024-04-05T03:20:32.079 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-05T03:20:32.095 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-05T03:20:32.109 INFO:tasks.cephadm:Teardown complete 2024-04-05T03:20:32.110 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-05T03:20:32.120 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-05T03:20:32.120 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-05T03:20:32.138 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-05T03:20:32.227 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-05T03:20:32.237 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-05T03:20:32.237 DEBUG:teuthology.orchestra.run.smithi033:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T03:20:32.239 DEBUG:teuthology.orchestra.run.smithi151:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T03:20:32.251 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-04-05T03:20:32.253 INFO:teuthology.orchestra.run.smithi151.stderr:bash: line 1: ntpq: command not found 2024-04-05T03:20:32.255 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T03:20:32.255 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-04-05T03:20:32.256 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 63 -1273us[-1281us] +/- 81ms 2024-04-05T03:20:32.256 INFO:teuthology.orchestra.run.smithi033.stdout:^* hv02.front.sepia.ceph.com 3 6 377 0 +263us[ +255us] +/- 70ms 2024-04-05T03:20:32.256 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 +832us[ +823us] +/- 71ms 2024-04-05T03:20:32.256 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T03:20:32.256 INFO:teuthology.orchestra.run.smithi151.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T03:20:32.257 INFO:teuthology.orchestra.run.smithi151.stdout:=============================================================================== 2024-04-05T03:20:32.257 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 64 -1310us[-1311us] +/- 81ms 2024-04-05T03:20:32.257 INFO:teuthology.orchestra.run.smithi151.stdout:^* hv02.front.sepia.ceph.com 3 6 377 63 +250us[ +249us] +/- 69ms 2024-04-05T03:20:32.257 INFO:teuthology.orchestra.run.smithi151.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 64 +837us[ +836us] +/- 70ms 2024-04-05T03:20:32.257 INFO:teuthology.orchestra.run.smithi151.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T03:20:32.258 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-05T03:20:32.268 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-05T03:20:32.269 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-05T03:20:32.278 DEBUG:teuthology.orchestra.run.smithi033:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-05T03:20:32.390 DEBUG:teuthology.orchestra.run.smithi151:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-05T03:20:32.467 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T03:20:32.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:20:32.498 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T03:20:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:20:32.529 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-05T03:20:32.539 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-05T03:20:32.550 INFO:teuthology.task.internal:Duration was 2359.266456 seconds 2024-04-05T03:20:32.551 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-05T03:20:32.561 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-05T03:20:32.561 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-05T03:20:32.564 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-05T03:20:32.604 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T03:20:32.612 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T03:20:33.022 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-05T03:20:33.022 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com 2024-04-05T03:20:33.022 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-05T03:20:33.081 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi151.front.sepia.ceph.com 2024-04-05T03:20:33.082 DEBUG:teuthology.orchestra.run.smithi151:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-05T03:20:33.105 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-05T03:20:33.105 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-05T03:20:33.124 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-05T03:20:33.174 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-05T03:20:33.174 DEBUG:teuthology.orchestra.run.smithi033:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-05T03:20:33.192 DEBUG:teuthology.orchestra.run.smithi151:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-05T03:20:33.660 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-05T03:20:33.670 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-05T03:20:33.671 DEBUG:teuthology.orchestra.run.smithi033:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-05T03:20:33.697 DEBUG:teuthology.orchestra.run.smithi151:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-05T03:20:33.724 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-05T03:20:33.737 DEBUG:teuthology.orchestra.run.smithi033:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:20:33.740 DEBUG:teuthology.orchestra.run.smithi151:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:20:33.762 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = core 2024-04-05T03:20:33.789 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = core 2024-04-05T03:20:33.836 DEBUG:teuthology.orchestra.run.smithi033:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:20:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:20:33.851 DEBUG:teuthology.orchestra.run.smithi151:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:20:33.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:20:33.890 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-05T03:20:33.904 INFO:teuthology.task.internal:Transferring archived files... 2024-04-05T03:20:33.905 DEBUG:teuthology.misc:Transferring archived files from smithi033:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239/remote/smithi033 2024-04-05T03:20:33.905 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-05T03:20:33.960 DEBUG:teuthology.misc:Transferring archived files from smithi151:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239/remote/smithi151 2024-04-05T03:20:33.961 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-05T03:20:34.001 INFO:teuthology.task.internal:Removing archive directory... 2024-04-05T03:20:34.001 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-05T03:20:34.004 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-05T03:20:34.050 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-05T03:20:34.064 INFO:teuthology.task.internal:Not uploading archives. 2024-04-05T03:20:34.064 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-05T03:20:34.075 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-05T03:20:34.075 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-05T03:20:34.077 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-05T03:20:34.089 INFO:teuthology.orchestra.run.smithi033.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 5 03:20 /home/ubuntu/cephtest 2024-04-05T03:20:34.104 INFO:teuthology.orchestra.run.smithi151.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 5 03:20 /home/ubuntu/cephtest 2024-04-05T03:20:34.105 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-05T03:20:34.118 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-05T03:20:34.172 INFO:teuthology.nuke:Checking targets against current locks 2024-04-05T03:20:34.200 DEBUG:teuthology.nuke:shortname: smithi033 2024-04-05T03:20:34.201 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-05T03:20:34.228 DEBUG:teuthology.nuke:shortname: smithi151 2024-04-05T03:20:34.228 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-05T03:20:34.232 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi033.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:32:33.016485', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGb6Z0f1svxcqtuGH/EwOxinIHa3wn5YuwPddKd3VEnYkXq3N6mhfHs3NKcYjqW65FVgdghso4GaRJxivQAwOJc='} 2024-04-05T03:20:34.258 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641239', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:32:33.017904', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCFjpiIx5qJZ+E0EHmLE7Ann6VQXu5/03zhZiPK+z0IpNCeJN9EIUQ8EFqAWSt1Dou+1kGAzgxWvEmPt4nFxbCU='} 2024-04-05T03:20:34.280 INFO:teuthology.orchestra.console.smithi033:Power off 2024-04-05T03:20:34.281 DEBUG:teuthology.orchestra.console.smithi033:pexpect command: ipmitool -H smithi033.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-05T03:20:34.304 DEBUG:teuthology.orchestra.console.smithi033:power off output: Chassis Power Control: Down/Off 2024-04-05T03:20:34.304 DEBUG:teuthology.orchestra.console.smithi033:pexpect command: ipmitool -H smithi033.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:20:34.323 INFO:teuthology.orchestra.console.smithi151:Power off 2024-04-05T03:20:34.323 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-05T03:20:34.333 DEBUG:teuthology.orchestra.console.smithi033:check power output: Chassis Power is on 2024-04-05T03:20:34.346 DEBUG:teuthology.orchestra.console.smithi151:power off output: Chassis Power Control: Down/Off 2024-04-05T03:20:34.346 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:20:34.369 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2024-04-05T03:20:38.335 DEBUG:teuthology.orchestra.console.smithi033:pexpect command: ipmitool -H smithi033.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:20:38.370 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:20:38.447 DEBUG:teuthology.orchestra.console.smithi033:check power output: Chassis Power is on 2024-04-05T03:20:38.481 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2024-04-05T03:20:42.452 DEBUG:teuthology.orchestra.console.smithi033:pexpect command: ipmitool -H smithi033.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:20:42.482 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:20:42.563 DEBUG:teuthology.orchestra.console.smithi033:check power output: Chassis Power is on 2024-04-05T03:20:42.592 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is off 2024-04-05T03:20:42.693 INFO:teuthology.orchestra.console.smithi151:Power off completed 2024-04-05T03:20:46.567 DEBUG:teuthology.orchestra.console.smithi033:pexpect command: ipmitool -H smithi033.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:20:46.677 DEBUG:teuthology.orchestra.console.smithi033:check power output: Chassis Power is off 2024-04-05T03:20:46.778 INFO:teuthology.orchestra.console.smithi033:Power off completed 2024-04-05T03:20:46.883 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 2359.2664561271667 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=4a685f6e14724085a068ed25a2383fb4 status: fail success: false 2024-04-05T03:20:46.883 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-05T03:20:46.963 INFO:teuthology.run:FAIL